{"level":"info","ts":"2026-09-18T09:37:39.171Z","logger":"setup","msg":"Loaded feature gates","featureGates":{"AsyncJobInfoQuery":false,"RayClusterStatusConditions":true,"RayCronJob":false,"RayJobDeletionPolicy":true,"RayMultiHostIndexing":true,"RayServiceIncrementalUpgrade":false}}
{"level":"info","ts":"2026-09-18T09:37:39.171Z","logger":"setup","msg":"Flag watchNamespace is not set. Watch custom resources in all namespaces."}
{"level":"info","ts":"2026-09-18T09:37:39.171Z","logger":"setup","msg":"Setup manager"}
{"level":"info","ts":"2026-09-18T09:37:39.466Z","logger":"tls","msg":"TLS profile watcher registered"}
{"level":"info","ts":"2026-09-18T09:37:39.480Z","logger":"setup","msg":"NetworkPolicy controller registered (annotation-based activation)"}
{"level":"info","ts":"2026-09-18T09:37:39.480Z","logger":"controller-runtime.builder","msg":"Registering a mutating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-09-18T09:37:39.480Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-09-18T09:37:39.481Z","logger":"controller-runtime.builder","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-09-18T09:37:39.481Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-09-18T09:37:39.481Z","logger":"setup","msg":"RayCronJob feature gate is disabled, skipping RayCronJob controller setup"}
{"level":"info","ts":"2026-09-18T09:37:39.481Z","logger":"setup","msg":"starting manager"}
{"level":"info","ts":"2026-09-18T09:37:39.481Z","logger":"controller-runtime.metrics","msg":"Starting metrics server"}
{"level":"info","ts":"2026-09-18T09:37:39.481Z","logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate","cert":"/etc/metrics-certs/tls.crt","key":"/etc/metrics-certs/tls.key"}
{"level":"info","ts":"2026-09-18T09:37:39.481Z","logger":"controller-runtime.metrics","msg":"Serving metrics server","bindAddress":":8080","secure":true}
{"level":"info","ts":"2026-09-18T09:37:39.481Z","msg":"starting server","name":"health probe","addr":"[::]:8082"}
{"level":"info","ts":"2026-09-18T09:37:39.481Z","logger":"controller-runtime.certwatcher","msg":"Starting certificate poll+watcher","cert":"/etc/metrics-certs/tls.crt","key":"/etc/metrics-certs/tls.key","interval":10}
{"level":"info","ts":"2026-09-18T09:37:39.481Z","logger":"controller-runtime.webhook","msg":"Starting webhook server"}
{"level":"info","ts":"2026-09-18T09:37:39.482Z","logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key"}
{"level":"info","ts":"2026-09-18T09:37:39.482Z","logger":"controller-runtime.webhook","msg":"Serving webhook server","host":"","port":9443}
{"level":"info","ts":"2026-09-18T09:37:39.482Z","logger":"controller-runtime.certwatcher","msg":"Starting certificate poll+watcher","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key","interval":10}
{"level":"info","ts":"2026-09-18T09:37:39.682Z","msg":"Starting EventSource","controller":"tls-profile-watcher","source":"kind source: *v1.APIServer"}
{"level":"info","ts":"2026-09-18T09:37:39.682Z","msg":"Attempting to acquire leader lease...","lock":"opendatahub/ray-operator-leader"}
{"level":"info","ts":"2026-09-18T09:37:39.691Z","msg":"Successfully acquired lease","lock":"opendatahub/ray-operator-leader"}
{"level":"info","ts":"2026-09-18T09:37:39.691Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Secret"}
{"level":"info","ts":"2026-09-18T09:37:39.691Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-09-18T09:37:39.691Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.Job"}
{"level":"info","ts":"2026-09-18T09:37:39.691Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-09-18T09:37:39.691Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-09-18T09:37:39.692Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-09-18T09:37:39.692Z","msg":"Starting EventSource","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.NetworkPolicy"}
{"level":"info","ts":"2026-09-18T09:37:39.692Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.RayJob"}
{"level":"info","ts":"2026-09-18T09:37:39.692Z","msg":"Starting EventSource","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-09-18T09:37:39.692Z","msg":"Starting EventSource","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-09-18T09:37:39.693Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.Route"}
{"level":"info","ts":"2026-09-18T09:37:39.693Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-09-18T09:37:39.693Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.ServiceAccount"}
{"level":"info","ts":"2026-09-18T09:37:39.693Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-09-18T09:37:39.765Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.RayService"}
{"level":"info","ts":"2026-09-18T09:37:39.765Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-09-18T09:37:39.766Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-09-18T09:37:39.766Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-09-18T09:37:40.267Z","logger":"controllers.RayService","msg":"Starting Controller"}
{"level":"info","ts":"2026-09-18T09:37:40.267Z","logger":"controllers.RayService","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-09-18T09:37:40.269Z","msg":"Starting Controller","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-09-18T09:37:40.269Z","logger":"controllers.RayCluster","msg":"Starting Controller"}
{"level":"info","ts":"2026-09-18T09:37:40.269Z","logger":"controllers.RayCluster","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-09-18T09:37:40.269Z","msg":"Starting Controller","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-09-18T09:37:40.269Z","msg":"Starting workers","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-09-18T09:37:40.269Z","msg":"Starting workers","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-09-18T09:37:40.269Z","msg":"Starting Controller","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-09-18T09:37:40.269Z","msg":"Starting workers","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-09-18T09:37:40.270Z","logger":"controllers.RayJob","msg":"Starting Controller"}
{"level":"info","ts":"2026-09-18T09:37:40.270Z","logger":"controllers.RayJob","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-09-18T09:37:40.270Z","msg":"Starting Controller","controller":"tls-profile-watcher"}
{"level":"info","ts":"2026-09-18T09:37:40.270Z","msg":"Starting workers","controller":"tls-profile-watcher","worker count":1}
{"level":"info","ts":"2026-09-18T09:40:06.230Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.231Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-r8v9f"}
{"level":"info","ts":"2026-09-18T09:40:06.240Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"97455fbc-2c71-4af9-93ab-65b55f130b99","namespacedName":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:40:06.240Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"97455fbc-2c71-4af9-93ab-65b55f130b99","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.240Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"97455fbc-2c71-4af9-93ab-65b55f130b99","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.240Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"97455fbc-2c71-4af9-93ab-65b55f130b99","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.240Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0c31788-f661-4413-96cc-024839ee98f8"}
{"level":"info","ts":"2026-09-18T09:40:06.240Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0c31788-f661-4413-96cc-024839ee98f8"}
{"level":"info","ts":"2026-09-18T09:40:06.240Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0c31788-f661-4413-96cc-024839ee98f8"}
{"level":"info","ts":"2026-09-18T09:40:06.241Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"40ed8628-8680-4975-82be-b596486e8f86","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.241Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"5c048141-6be6-43e5-9aae-5febede40ca6","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.246Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.246Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-r8v9f"}
{"level":"info","ts":"2026-09-18T09:40:06.266Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0c31788-f661-4413-96cc-024839ee98f8","name":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:06.266Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0c31788-f661-4413-96cc-024839ee98f8","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-09-18T09:40:06.266Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0c31788-f661-4413-96cc-024839ee98f8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.266Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0c31788-f661-4413-96cc-024839ee98f8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-09-18T09:40:06.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0c31788-f661-4413-96cc-024839ee98f8","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0c31788-f661-4413-96cc-024839ee98f8","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0c31788-f661-4413-96cc-024839ee98f8","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.266Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0c31788-f661-4413-96cc-024839ee98f8","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:40:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:40:06.274Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"5c048141-6be6-43e5-9aae-5febede40ca6","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.274Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"3f59d545-8ace-401b-b13c-cc4a108ad7cd","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.279Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0c31788-f661-4413-96cc-024839ee98f8","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-r8v9f","name":"raycluster"}}
{"level":"info","ts":"2026-09-18T09:40:06.279Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0c31788-f661-4413-96cc-024839ee98f8"}
{"level":"error","ts":"2026-09-18T09:40:06.279Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0c31788-f661-4413-96cc-024839ee98f8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.279Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7b8a3c37-56ff-473c-a578-644357bf90da"}
{"level":"info","ts":"2026-09-18T09:40:06.279Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7b8a3c37-56ff-473c-a578-644357bf90da"}
{"level":"info","ts":"2026-09-18T09:40:06.279Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7b8a3c37-56ff-473c-a578-644357bf90da"}
{"level":"info","ts":"2026-09-18T09:40:06.279Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7b8a3c37-56ff-473c-a578-644357bf90da","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:06.279Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7b8a3c37-56ff-473c-a578-644357bf90da","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-09-18T09:40:06.279Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7b8a3c37-56ff-473c-a578-644357bf90da","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.279Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7b8a3c37-56ff-473c-a578-644357bf90da","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-09-18T09:40:06.279Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7b8a3c37-56ff-473c-a578-644357bf90da","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.279Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7b8a3c37-56ff-473c-a578-644357bf90da","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.279Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7b8a3c37-56ff-473c-a578-644357bf90da","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.279Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7b8a3c37-56ff-473c-a578-644357bf90da","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:40:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:40:06.283Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"97455fbc-2c71-4af9-93ab-65b55f130b99","name":"raycluster-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:40:06.287Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"3f59d545-8ace-401b-b13c-cc4a108ad7cd","name":"raycluster-head"}
{"level":"info","ts":"2026-09-18T09:40:06.292Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7b8a3c37-56ff-473c-a578-644357bf90da"}
{"level":"error","ts":"2026-09-18T09:40:06.292Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7b8a3c37-56ff-473c-a578-644357bf90da","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.292Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"036ba61a-c909-4a4d-8cc5-e7aa9539ca00"}
{"level":"info","ts":"2026-09-18T09:40:06.292Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"036ba61a-c909-4a4d-8cc5-e7aa9539ca00"}
{"level":"info","ts":"2026-09-18T09:40:06.292Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"036ba61a-c909-4a4d-8cc5-e7aa9539ca00"}
{"level":"info","ts":"2026-09-18T09:40:06.292Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"036ba61a-c909-4a4d-8cc5-e7aa9539ca00","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:06.292Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"036ba61a-c909-4a4d-8cc5-e7aa9539ca00","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-09-18T09:40:06.292Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"036ba61a-c909-4a4d-8cc5-e7aa9539ca00","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.292Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"036ba61a-c909-4a4d-8cc5-e7aa9539ca00","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-09-18T09:40:06.292Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"036ba61a-c909-4a4d-8cc5-e7aa9539ca00","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.292Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"036ba61a-c909-4a4d-8cc5-e7aa9539ca00","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.292Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"036ba61a-c909-4a4d-8cc5-e7aa9539ca00","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.292Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"036ba61a-c909-4a4d-8cc5-e7aa9539ca00"}
{"level":"error","ts":"2026-09-18T09:40:06.292Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"036ba61a-c909-4a4d-8cc5-e7aa9539ca00","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.299Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"3f59d545-8ace-401b-b13c-cc4a108ad7cd","name":"raycluster-workers"}
{"level":"info","ts":"2026-09-18T09:40:06.299Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"3f59d545-8ace-401b-b13c-cc4a108ad7cd","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.299Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"2f7d62bb-21e7-43e6-8680-c6f13c1cc24e","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.303Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"43c0dfe3-0b37-4ded-8a9f-d2686a397c3b"}
{"level":"info","ts":"2026-09-18T09:40:06.303Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"43c0dfe3-0b37-4ded-8a9f-d2686a397c3b"}
{"level":"info","ts":"2026-09-18T09:40:06.303Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"43c0dfe3-0b37-4ded-8a9f-d2686a397c3b"}
{"level":"info","ts":"2026-09-18T09:40:06.303Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"43c0dfe3-0b37-4ded-8a9f-d2686a397c3b","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:06.303Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"43c0dfe3-0b37-4ded-8a9f-d2686a397c3b","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-09-18T09:40:06.303Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"43c0dfe3-0b37-4ded-8a9f-d2686a397c3b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.303Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"43c0dfe3-0b37-4ded-8a9f-d2686a397c3b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-09-18T09:40:06.303Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"43c0dfe3-0b37-4ded-8a9f-d2686a397c3b","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.303Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"43c0dfe3-0b37-4ded-8a9f-d2686a397c3b","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.303Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"43c0dfe3-0b37-4ded-8a9f-d2686a397c3b","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.303Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"43c0dfe3-0b37-4ded-8a9f-d2686a397c3b"}
{"level":"error","ts":"2026-09-18T09:40:06.303Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"43c0dfe3-0b37-4ded-8a9f-d2686a397c3b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.365Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"40ed8628-8680-4975-82be-b596486e8f86","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.366Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d654ef42-5710-45d9-802f-facb9984d9ad"}
{"level":"info","ts":"2026-09-18T09:40:06.366Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d654ef42-5710-45d9-802f-facb9984d9ad"}
{"level":"info","ts":"2026-09-18T09:40:06.366Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d654ef42-5710-45d9-802f-facb9984d9ad"}
{"level":"info","ts":"2026-09-18T09:40:06.366Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d654ef42-5710-45d9-802f-facb9984d9ad","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:06.366Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d654ef42-5710-45d9-802f-facb9984d9ad","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-09-18T09:40:06.366Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d654ef42-5710-45d9-802f-facb9984d9ad","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.366Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d654ef42-5710-45d9-802f-facb9984d9ad","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-09-18T09:40:06.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d654ef42-5710-45d9-802f-facb9984d9ad","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d654ef42-5710-45d9-802f-facb9984d9ad","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d654ef42-5710-45d9-802f-facb9984d9ad","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.366Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d654ef42-5710-45d9-802f-facb9984d9ad"}
{"level":"error","ts":"2026-09-18T09:40:06.366Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d654ef42-5710-45d9-802f-facb9984d9ad","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.366Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"2f7d62bb-21e7-43e6-8680-c6f13c1cc24e","name":"raycluster-head"}
{"level":"info","ts":"2026-09-18T09:40:06.380Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"2f7d62bb-21e7-43e6-8680-c6f13c1cc24e","name":"raycluster-workers"}
{"level":"info","ts":"2026-09-18T09:40:06.380Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"2f7d62bb-21e7-43e6-8680-c6f13c1cc24e","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.389Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"97455fbc-2c71-4af9-93ab-65b55f130b99","name":"kuberay-gateway-access","namespace":"test-ns-r8v9f","operation":"created"}
{"level":"info","ts":"2026-09-18T09:40:06.403Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"40ed8628-8680-4975-82be-b596486e8f86","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.446Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9fd1e8f-04c7-416e-a69a-b40f2cb2927a"}
{"level":"info","ts":"2026-09-18T09:40:06.446Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9fd1e8f-04c7-416e-a69a-b40f2cb2927a"}
{"level":"info","ts":"2026-09-18T09:40:06.446Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9fd1e8f-04c7-416e-a69a-b40f2cb2927a"}
{"level":"info","ts":"2026-09-18T09:40:06.446Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9fd1e8f-04c7-416e-a69a-b40f2cb2927a","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:06.446Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9fd1e8f-04c7-416e-a69a-b40f2cb2927a","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-09-18T09:40:06.446Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9fd1e8f-04c7-416e-a69a-b40f2cb2927a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.447Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9fd1e8f-04c7-416e-a69a-b40f2cb2927a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-09-18T09:40:06.447Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9fd1e8f-04c7-416e-a69a-b40f2cb2927a","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.447Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9fd1e8f-04c7-416e-a69a-b40f2cb2927a","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.447Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9fd1e8f-04c7-416e-a69a-b40f2cb2927a","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.447Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9fd1e8f-04c7-416e-a69a-b40f2cb2927a"}
{"level":"error","ts":"2026-09-18T09:40:06.447Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9fd1e8f-04c7-416e-a69a-b40f2cb2927a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.499Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"97455fbc-2c71-4af9-93ab-65b55f130b99","name":"test-ns-r8v9f-raycluster","operation":"created"}
{"level":"info","ts":"2026-09-18T09:40:06.504Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"40ed8628-8680-4975-82be-b596486e8f86","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.766Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0f6ef71-13a5-49be-9428-d1dcd72fd619"}
{"level":"info","ts":"2026-09-18T09:40:06.766Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0f6ef71-13a5-49be-9428-d1dcd72fd619"}
{"level":"info","ts":"2026-09-18T09:40:06.766Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0f6ef71-13a5-49be-9428-d1dcd72fd619"}
{"level":"info","ts":"2026-09-18T09:40:06.766Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0f6ef71-13a5-49be-9428-d1dcd72fd619","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:06.766Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0f6ef71-13a5-49be-9428-d1dcd72fd619","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-09-18T09:40:06.766Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0f6ef71-13a5-49be-9428-d1dcd72fd619","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.766Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0f6ef71-13a5-49be-9428-d1dcd72fd619","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-09-18T09:40:06.767Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0f6ef71-13a5-49be-9428-d1dcd72fd619","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.767Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0f6ef71-13a5-49be-9428-d1dcd72fd619","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.767Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0f6ef71-13a5-49be-9428-d1dcd72fd619","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:06.767Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0f6ef71-13a5-49be-9428-d1dcd72fd619"}
{"level":"error","ts":"2026-09-18T09:40:06.767Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0f6ef71-13a5-49be-9428-d1dcd72fd619","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.767Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"40ed8628-8680-4975-82be-b596486e8f86","rayCluster":"raycluster","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T09:40:06.767Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"40ed8628-8680-4975-82be-b596486e8f86","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.776Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"40ed8628-8680-4975-82be-b596486e8f86","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.788Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"40ed8628-8680-4975-82be-b596486e8f86","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:40:06.797Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"40ed8628-8680-4975-82be-b596486e8f86","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:40:06.797Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"40ed8628-8680-4975-82be-b596486e8f86"}
{"level":"info","ts":"2026-09-18T09:40:06.797Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"221ec07e-3b59-4a8a-a856-266b83a852e3","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.797Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"221ec07e-3b59-4a8a-a856-266b83a852e3"}
{"level":"info","ts":"2026-09-18T09:40:06.874Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"97455fbc-2c71-4af9-93ab-65b55f130b99","name":"kube-rbac-proxy-config-raycluster","operation":"created"}
{"level":"error","ts":"2026-09-18T09:40:06.882Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"97455fbc-2c71-4af9-93ab-65b55f130b99","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.882Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"97455fbc-2c71-4af9-93ab-65b55f130b99"}
{"level":"error","ts":"2026-09-18T09:40:06.882Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"97455fbc-2c71-4af9-93ab-65b55f130b99","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.882Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"dc26761f-0040-46e5-89ac-5340f428f18c","namespacedName":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:40:06.882Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"dc26761f-0040-46e5-89ac-5340f428f18c","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.882Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"dc26761f-0040-46e5-89ac-5340f428f18c","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.896Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"dc26761f-0040-46e5-89ac-5340f428f18c","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.896Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"b6afcccf-6460-4b96-83ce-a71cbd380ead","namespacedName":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:40:06.896Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"b6afcccf-6460-4b96-83ce-a71cbd380ead","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.896Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"b6afcccf-6460-4b96-83ce-a71cbd380ead","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.896Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f371ef6c-e5db-49e1-9b3b-dd497dc03cdc","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.896Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f371ef6c-e5db-49e1-9b3b-dd497dc03cdc"}
{"level":"info","ts":"2026-09-18T09:40:06.896Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"840d9908-0b4a-4c82-b78d-159d54ec9490","cluster":"raycluster"}
{"level":"error","ts":"2026-09-18T09:40:06.904Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"b6afcccf-6460-4b96-83ce-a71cbd380ead","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.905Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"b6afcccf-6460-4b96-83ce-a71cbd380ead"}
{"level":"error","ts":"2026-09-18T09:40:06.905Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"b6afcccf-6460-4b96-83ce-a71cbd380ead","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:06.910Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"ffb3a937-f428-40be-a2b6-0ab27d158dc3","namespacedName":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:40:06.910Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"840d9908-0b4a-4c82-b78d-159d54ec9490","name":"raycluster-head"}
{"level":"info","ts":"2026-09-18T09:40:06.910Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"ffb3a937-f428-40be-a2b6-0ab27d158dc3","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.910Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"ffb3a937-f428-40be-a2b6-0ab27d158dc3","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.924Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"ffb3a937-f428-40be-a2b6-0ab27d158dc3","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:06.966Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"840d9908-0b4a-4c82-b78d-159d54ec9490","name":"raycluster-workers"}
{"level":"info","ts":"2026-09-18T09:40:06.966Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"840d9908-0b4a-4c82-b78d-159d54ec9490","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:07.088Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"64c1b28b-fb1c-4bc0-bb5b-b404e3c2bf6c"}
{"level":"info","ts":"2026-09-18T09:40:07.088Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"64c1b28b-fb1c-4bc0-bb5b-b404e3c2bf6c"}
{"level":"info","ts":"2026-09-18T09:40:07.088Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"64c1b28b-fb1c-4bc0-bb5b-b404e3c2bf6c"}
{"level":"info","ts":"2026-09-18T09:40:07.088Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"64c1b28b-fb1c-4bc0-bb5b-b404e3c2bf6c","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:07.088Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"64c1b28b-fb1c-4bc0-bb5b-b404e3c2bf6c","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-09-18T09:40:07.088Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"64c1b28b-fb1c-4bc0-bb5b-b404e3c2bf6c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:07.088Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"64c1b28b-fb1c-4bc0-bb5b-b404e3c2bf6c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-09-18T09:40:07.088Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"64c1b28b-fb1c-4bc0-bb5b-b404e3c2bf6c","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:07.088Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"64c1b28b-fb1c-4bc0-bb5b-b404e3c2bf6c","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:07.088Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"64c1b28b-fb1c-4bc0-bb5b-b404e3c2bf6c","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:07.088Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"64c1b28b-fb1c-4bc0-bb5b-b404e3c2bf6c"}
{"level":"error","ts":"2026-09-18T09:40:07.088Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"64c1b28b-fb1c-4bc0-bb5b-b404e3c2bf6c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:07.729Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"79bcd56b-fcc7-4d5e-891f-cdf79086c725"}
{"level":"info","ts":"2026-09-18T09:40:07.729Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"79bcd56b-fcc7-4d5e-891f-cdf79086c725"}
{"level":"info","ts":"2026-09-18T09:40:07.729Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"79bcd56b-fcc7-4d5e-891f-cdf79086c725"}
{"level":"info","ts":"2026-09-18T09:40:07.729Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"79bcd56b-fcc7-4d5e-891f-cdf79086c725","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:07.729Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"79bcd56b-fcc7-4d5e-891f-cdf79086c725","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-09-18T09:40:07.729Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"79bcd56b-fcc7-4d5e-891f-cdf79086c725","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:07.729Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"79bcd56b-fcc7-4d5e-891f-cdf79086c725","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-09-18T09:40:07.729Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"79bcd56b-fcc7-4d5e-891f-cdf79086c725","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:07.729Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"79bcd56b-fcc7-4d5e-891f-cdf79086c725","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:07.729Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"79bcd56b-fcc7-4d5e-891f-cdf79086c725","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:40:07.729Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"79bcd56b-fcc7-4d5e-891f-cdf79086c725"}
{"level":"error","ts":"2026-09-18T09:40:07.729Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"79bcd56b-fcc7-4d5e-891f-cdf79086c725","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:09.010Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759"}
{"level":"info","ts":"2026-09-18T09:40:09.010Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759"}
{"level":"info","ts":"2026-09-18T09:40:09.010Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759"}
{"level":"info","ts":"2026-09-18T09:40:09.010Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:09.010Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759"}
{"level":"info","ts":"2026-09-18T09:40:09.010Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:40:09.010Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759"}
{"level":"info","ts":"2026-09-18T09:40:09.010Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:09.010Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","cluster":"raycluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:40:09.010Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","cluster":"raycluster","authType":"IntegratedOAuth","serviceAccount":"raycluster-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:40:09.010Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster","ray.io/group":"headgroup","ray.io/identifier":"raycluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:40:09.010Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:40:09.010Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:40:09.010Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:40:09.010Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","enabled":true}
{"level":"info","ts":"2026-09-18T09:40:09.030Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","name":"raycluster-head-2vjtt"}
{"level":"info","ts":"2026-09-18T09:40:09.030Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:40:09.030Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:09.030Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:40:09.030Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:40:09.031Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:09.031Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:40:09.031Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759"}
{"level":"info","ts":"2026-09-18T09:40:09.031Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:40:09.031Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:40:09.031Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:40:09.031Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","enabled":true}
{"level":"info","ts":"2026-09-18T09:40:09.047Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","name":"raycluster-small-group-worker-cjnn9"}
{"level":"info","ts":"2026-09-18T09:40:09.048Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:09.048Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:09.048Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"221bcbc8-1797-45fc-8d4d-d8ee1e2da759","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:40:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:40:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:40:09.061Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386"}
{"level":"info","ts":"2026-09-18T09:40:09.061Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386"}
{"level":"info","ts":"2026-09-18T09:40:09.061Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386"}
{"level":"info","ts":"2026-09-18T09:40:09.061Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:09.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:40:09.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:40:09.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:40:09.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:40:09.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:09.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:40:09.061Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:40:09.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:09.061Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:09.062Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:09.062Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:40:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:40:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:40:09.062Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"e14b760d-40fd-4161-b03d-882e13c806be","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:09.062Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"e14b760d-40fd-4161-b03d-882e13c806be","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:40:09.062Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"4653605c-bba3-4567-ae43-3142e1b6b694","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:09.071Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-r8v9f","name":"raycluster"}}
{"level":"info","ts":"2026-09-18T09:40:09.071Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386"}
{"level":"error","ts":"2026-09-18T09:40:09.071Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"21889fbb-2df7-4125-963d-c0ed80736386","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:40:09.075Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7cb846fd-7d77-4e7d-b0ee-46a571ac7e3f"}
{"level":"info","ts":"2026-09-18T09:40:09.075Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7cb846fd-7d77-4e7d-b0ee-46a571ac7e3f"}
{"level":"info","ts":"2026-09-18T09:40:09.075Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7cb846fd-7d77-4e7d-b0ee-46a571ac7e3f"}
{"level":"info","ts":"2026-09-18T09:40:09.075Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7cb846fd-7d77-4e7d-b0ee-46a571ac7e3f","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:09.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7cb846fd-7d77-4e7d-b0ee-46a571ac7e3f","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:40:09.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7cb846fd-7d77-4e7d-b0ee-46a571ac7e3f","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:40:09.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7cb846fd-7d77-4e7d-b0ee-46a571ac7e3f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:40:09.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7cb846fd-7d77-4e7d-b0ee-46a571ac7e3f","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:40:09.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7cb846fd-7d77-4e7d-b0ee-46a571ac7e3f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:09.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7cb846fd-7d77-4e7d-b0ee-46a571ac7e3f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:40:09.075Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7cb846fd-7d77-4e7d-b0ee-46a571ac7e3f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:40:09.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7cb846fd-7d77-4e7d-b0ee-46a571ac7e3f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:09.075Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7cb846fd-7d77-4e7d-b0ee-46a571ac7e3f","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:09.076Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7cb846fd-7d77-4e7d-b0ee-46a571ac7e3f","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:09.076Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7cb846fd-7d77-4e7d-b0ee-46a571ac7e3f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:40:09.076Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7cb846fd-7d77-4e7d-b0ee-46a571ac7e3f","seconds":300}
{"level":"info","ts":"2026-09-18T09:40:09.076Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"3da37726-34b6-4ee1-9b87-844334147e67"}
{"level":"info","ts":"2026-09-18T09:40:09.076Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"3da37726-34b6-4ee1-9b87-844334147e67"}
{"level":"info","ts":"2026-09-18T09:40:09.076Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"3da37726-34b6-4ee1-9b87-844334147e67"}
{"level":"info","ts":"2026-09-18T09:40:09.076Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"3da37726-34b6-4ee1-9b87-844334147e67","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:09.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"3da37726-34b6-4ee1-9b87-844334147e67","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:40:09.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"3da37726-34b6-4ee1-9b87-844334147e67","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:40:09.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"3da37726-34b6-4ee1-9b87-844334147e67","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:40:09.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"3da37726-34b6-4ee1-9b87-844334147e67","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:40:09.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"3da37726-34b6-4ee1-9b87-844334147e67","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:09.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"3da37726-34b6-4ee1-9b87-844334147e67","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:40:09.076Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"3da37726-34b6-4ee1-9b87-844334147e67","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:40:09.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"3da37726-34b6-4ee1-9b87-844334147e67","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:09.077Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"3da37726-34b6-4ee1-9b87-844334147e67","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:09.077Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"3da37726-34b6-4ee1-9b87-844334147e67","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:09.077Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"3da37726-34b6-4ee1-9b87-844334147e67","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:40:09.077Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"3da37726-34b6-4ee1-9b87-844334147e67","seconds":300}
{"level":"info","ts":"2026-09-18T09:40:09.082Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"4653605c-bba3-4567-ae43-3142e1b6b694","name":"raycluster-head"}
{"level":"info","ts":"2026-09-18T09:40:09.098Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"4653605c-bba3-4567-ae43-3142e1b6b694","name":"raycluster-workers"}
{"level":"info","ts":"2026-09-18T09:40:09.098Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"4653605c-bba3-4567-ae43-3142e1b6b694","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:09.100Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"eda8936a-0a9b-49bc-97a9-afd90fcbbe67"}
{"level":"info","ts":"2026-09-18T09:40:09.100Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"eda8936a-0a9b-49bc-97a9-afd90fcbbe67"}
{"level":"info","ts":"2026-09-18T09:40:09.100Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"eda8936a-0a9b-49bc-97a9-afd90fcbbe67"}
{"level":"info","ts":"2026-09-18T09:40:09.100Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"eda8936a-0a9b-49bc-97a9-afd90fcbbe67","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:09.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"eda8936a-0a9b-49bc-97a9-afd90fcbbe67","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:40:09.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"eda8936a-0a9b-49bc-97a9-afd90fcbbe67","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:40:09.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"eda8936a-0a9b-49bc-97a9-afd90fcbbe67","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:40:09.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"eda8936a-0a9b-49bc-97a9-afd90fcbbe67","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:40:09.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"eda8936a-0a9b-49bc-97a9-afd90fcbbe67","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:09.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"eda8936a-0a9b-49bc-97a9-afd90fcbbe67","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:40:09.100Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"eda8936a-0a9b-49bc-97a9-afd90fcbbe67","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:40:09.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"eda8936a-0a9b-49bc-97a9-afd90fcbbe67","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:09.100Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"eda8936a-0a9b-49bc-97a9-afd90fcbbe67","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:09.100Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"eda8936a-0a9b-49bc-97a9-afd90fcbbe67","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:09.100Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"eda8936a-0a9b-49bc-97a9-afd90fcbbe67","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:40:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:40:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:40:09.112Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"bbcf19be-0dd7-4c99-83df-cde14cc01b02","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:09.112Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f41ee0b7-1ee5-4dd5-8a91-f454b0931c74","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:09.112Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f41ee0b7-1ee5-4dd5-8a91-f454b0931c74","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:40:09.122Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"00c52a06-96e3-4874-9896-6703fcbbedbe"}
{"level":"info","ts":"2026-09-18T09:40:09.122Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"00c52a06-96e3-4874-9896-6703fcbbedbe"}
{"level":"info","ts":"2026-09-18T09:40:09.122Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"00c52a06-96e3-4874-9896-6703fcbbedbe"}
{"level":"info","ts":"2026-09-18T09:40:09.122Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"00c52a06-96e3-4874-9896-6703fcbbedbe","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:09.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"00c52a06-96e3-4874-9896-6703fcbbedbe","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:40:09.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"00c52a06-96e3-4874-9896-6703fcbbedbe","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:40:09.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"00c52a06-96e3-4874-9896-6703fcbbedbe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:40:09.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"00c52a06-96e3-4874-9896-6703fcbbedbe","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:40:09.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"00c52a06-96e3-4874-9896-6703fcbbedbe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:09.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"00c52a06-96e3-4874-9896-6703fcbbedbe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:40:09.123Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"00c52a06-96e3-4874-9896-6703fcbbedbe","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:40:09.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"00c52a06-96e3-4874-9896-6703fcbbedbe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:09.123Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"00c52a06-96e3-4874-9896-6703fcbbedbe","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:09.123Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"00c52a06-96e3-4874-9896-6703fcbbedbe","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:09.123Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"00c52a06-96e3-4874-9896-6703fcbbedbe","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:40:09.123Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"00c52a06-96e3-4874-9896-6703fcbbedbe","seconds":300}
{"level":"info","ts":"2026-09-18T09:40:09.166Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"bbcf19be-0dd7-4c99-83df-cde14cc01b02","name":"raycluster-head"}
{"level":"info","ts":"2026-09-18T09:40:09.179Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"bbcf19be-0dd7-4c99-83df-cde14cc01b02","name":"raycluster-workers"}
{"level":"info","ts":"2026-09-18T09:40:09.179Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"bbcf19be-0dd7-4c99-83df-cde14cc01b02","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:09.641Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"748f748e-8d13-4960-bf72-b701792b0d1e"}
{"level":"info","ts":"2026-09-18T09:40:09.641Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"748f748e-8d13-4960-bf72-b701792b0d1e"}
{"level":"info","ts":"2026-09-18T09:40:09.641Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"748f748e-8d13-4960-bf72-b701792b0d1e"}
{"level":"info","ts":"2026-09-18T09:40:09.641Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"748f748e-8d13-4960-bf72-b701792b0d1e","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:09.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"748f748e-8d13-4960-bf72-b701792b0d1e","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:40:09.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"748f748e-8d13-4960-bf72-b701792b0d1e","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:40:09.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"748f748e-8d13-4960-bf72-b701792b0d1e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:40:09.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"748f748e-8d13-4960-bf72-b701792b0d1e","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:40:09.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"748f748e-8d13-4960-bf72-b701792b0d1e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:09.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"748f748e-8d13-4960-bf72-b701792b0d1e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:40:09.641Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"748f748e-8d13-4960-bf72-b701792b0d1e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:40:09.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"748f748e-8d13-4960-bf72-b701792b0d1e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:09.641Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"748f748e-8d13-4960-bf72-b701792b0d1e","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:09.641Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"748f748e-8d13-4960-bf72-b701792b0d1e","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:09.641Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"748f748e-8d13-4960-bf72-b701792b0d1e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:40:09.641Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"748f748e-8d13-4960-bf72-b701792b0d1e","seconds":300}
{"level":"info","ts":"2026-09-18T09:40:09.644Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ab66c190-7a78-4132-8128-bda1e3172621"}
{"level":"info","ts":"2026-09-18T09:40:09.644Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ab66c190-7a78-4132-8128-bda1e3172621"}
{"level":"info","ts":"2026-09-18T09:40:09.644Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ab66c190-7a78-4132-8128-bda1e3172621"}
{"level":"info","ts":"2026-09-18T09:40:09.644Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ab66c190-7a78-4132-8128-bda1e3172621","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:09.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ab66c190-7a78-4132-8128-bda1e3172621","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:40:09.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ab66c190-7a78-4132-8128-bda1e3172621","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:40:09.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ab66c190-7a78-4132-8128-bda1e3172621","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:40:09.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ab66c190-7a78-4132-8128-bda1e3172621","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:40:09.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ab66c190-7a78-4132-8128-bda1e3172621","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:09.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ab66c190-7a78-4132-8128-bda1e3172621","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:40:09.644Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ab66c190-7a78-4132-8128-bda1e3172621","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:40:09.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ab66c190-7a78-4132-8128-bda1e3172621","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:09.644Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ab66c190-7a78-4132-8128-bda1e3172621","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:09.644Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ab66c190-7a78-4132-8128-bda1e3172621","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:09.645Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ab66c190-7a78-4132-8128-bda1e3172621","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:40:09.645Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ab66c190-7a78-4132-8128-bda1e3172621","seconds":300}
{"level":"info","ts":"2026-09-18T09:40:11.114Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ff1a5597-13f0-42be-910d-6f07483cb712"}
{"level":"info","ts":"2026-09-18T09:40:11.114Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ff1a5597-13f0-42be-910d-6f07483cb712"}
{"level":"info","ts":"2026-09-18T09:40:11.114Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ff1a5597-13f0-42be-910d-6f07483cb712"}
{"level":"info","ts":"2026-09-18T09:40:11.114Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ff1a5597-13f0-42be-910d-6f07483cb712","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:40:11.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ff1a5597-13f0-42be-910d-6f07483cb712","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:40:11.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ff1a5597-13f0-42be-910d-6f07483cb712","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:40:11.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ff1a5597-13f0-42be-910d-6f07483cb712","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:40:11.115Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ff1a5597-13f0-42be-910d-6f07483cb712","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:40:11.115Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ff1a5597-13f0-42be-910d-6f07483cb712","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:11.115Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ff1a5597-13f0-42be-910d-6f07483cb712","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:40:11.115Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ff1a5597-13f0-42be-910d-6f07483cb712","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:40:11.115Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ff1a5597-13f0-42be-910d-6f07483cb712","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:40:11.115Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ff1a5597-13f0-42be-910d-6f07483cb712","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:11.115Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ff1a5597-13f0-42be-910d-6f07483cb712","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:40:11.115Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ff1a5597-13f0-42be-910d-6f07483cb712","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:40:11.115Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ff1a5597-13f0-42be-910d-6f07483cb712","seconds":300}
{"level":"info","ts":"2026-09-18T09:40:36.798Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"ddd1b091-700f-4009-b90d-fea01c9559b6","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:40:36.798Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"ddd1b091-700f-4009-b90d-fea01c9559b6","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:41:36.799Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"1822a552-0958-4553-9cf6-eacb36ab1253","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:41:36.799Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"1822a552-0958-4553-9cf6-eacb36ab1253","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:42:36.800Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"eac19f79-4804-401d-9868-0038220ac579","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:42:36.800Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"eac19f79-4804-401d-9868-0038220ac579","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:43:07.380Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9f00d48f-e2c7-49c8-9465-09c04014d7dc"}
{"level":"info","ts":"2026-09-18T09:43:07.380Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9f00d48f-e2c7-49c8-9465-09c04014d7dc"}
{"level":"info","ts":"2026-09-18T09:43:07.380Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9f00d48f-e2c7-49c8-9465-09c04014d7dc"}
{"level":"info","ts":"2026-09-18T09:43:07.380Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9f00d48f-e2c7-49c8-9465-09c04014d7dc","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:43:07.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9f00d48f-e2c7-49c8-9465-09c04014d7dc","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:43:07.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9f00d48f-e2c7-49c8-9465-09c04014d7dc","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:43:07.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9f00d48f-e2c7-49c8-9465-09c04014d7dc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:43:07.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9f00d48f-e2c7-49c8-9465-09c04014d7dc","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:43:07.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9f00d48f-e2c7-49c8-9465-09c04014d7dc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:43:07.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9f00d48f-e2c7-49c8-9465-09c04014d7dc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:43:07.380Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9f00d48f-e2c7-49c8-9465-09c04014d7dc","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:43:07.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9f00d48f-e2c7-49c8-9465-09c04014d7dc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:43:07.380Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9f00d48f-e2c7-49c8-9465-09c04014d7dc","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:43:07.380Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9f00d48f-e2c7-49c8-9465-09c04014d7dc","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:43:07.380Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9f00d48f-e2c7-49c8-9465-09c04014d7dc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:43:07.380Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9f00d48f-e2c7-49c8-9465-09c04014d7dc","seconds":300}
{"level":"info","ts":"2026-09-18T09:43:08.536Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"dcaeecc3-71c7-4d78-aa18-c36e3087385d"}
{"level":"info","ts":"2026-09-18T09:43:08.536Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"dcaeecc3-71c7-4d78-aa18-c36e3087385d"}
{"level":"info","ts":"2026-09-18T09:43:08.536Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"dcaeecc3-71c7-4d78-aa18-c36e3087385d"}
{"level":"info","ts":"2026-09-18T09:43:08.536Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"dcaeecc3-71c7-4d78-aa18-c36e3087385d","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:43:08.536Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"dcaeecc3-71c7-4d78-aa18-c36e3087385d","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:43:08.536Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"dcaeecc3-71c7-4d78-aa18-c36e3087385d","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:43:08.536Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"dcaeecc3-71c7-4d78-aa18-c36e3087385d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:43:08.536Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"dcaeecc3-71c7-4d78-aa18-c36e3087385d","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:43:08.536Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"dcaeecc3-71c7-4d78-aa18-c36e3087385d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:43:08.536Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"dcaeecc3-71c7-4d78-aa18-c36e3087385d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:43:08.536Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"dcaeecc3-71c7-4d78-aa18-c36e3087385d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:43:08.536Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"dcaeecc3-71c7-4d78-aa18-c36e3087385d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:43:08.536Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"dcaeecc3-71c7-4d78-aa18-c36e3087385d","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:43:08.536Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"dcaeecc3-71c7-4d78-aa18-c36e3087385d","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:43:08.536Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"dcaeecc3-71c7-4d78-aa18-c36e3087385d","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:40:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:43:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:43:08.548Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f14408af-0c62-44e9-8f02-aaf094a99a00","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:43:08.548Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f14408af-0c62-44e9-8f02-aaf094a99a00","pod":"raycluster-head-2vjtt","ip":"10.132.0.30"}
{"level":"info","ts":"2026-09-18T09:43:08.548Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f14408af-0c62-44e9-8f02-aaf094a99a00","pod":"raycluster-small-group-worker-cjnn9","ip":"10.133.0.25"}
{"level":"info","ts":"2026-09-18T09:43:08.548Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"c01d2b3c-6168-4b26-b3f2-eb12d11a68f7","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:43:08.574Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"c01d2b3c-6168-4b26-b3f2-eb12d11a68f7","name":"raycluster-head"}
{"level":"info","ts":"2026-09-18T09:43:08.575Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f14408af-0c62-44e9-8f02-aaf094a99a00","certificate":"ray-head-cert-raycluster","dnsNames":["raycluster-head-svc","localhost","raycluster-head-svc.test-ns-r8v9f.svc","raycluster-head-svc.test-ns-r8v9f.svc.cluster.local"],"ipAddresses":["10.132.0.30","10.133.0.25","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:43:08.588Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"c01d2b3c-6168-4b26-b3f2-eb12d11a68f7","name":"raycluster-workers"}
{"level":"info","ts":"2026-09-18T09:43:08.588Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"c01d2b3c-6168-4b26-b3f2-eb12d11a68f7","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:43:08.588Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f14408af-0c62-44e9-8f02-aaf094a99a00","certificate":"ray-worker-cert-raycluster","dnsNames":["raycluster-worker-svc","localhost","raycluster-small-group","raycluster-worker-svc.test-ns-r8v9f.svc","raycluster-worker-svc.test-ns-r8v9f.svc.cluster.local","*.raycluster-worker-svc.test-ns-r8v9f.svc","*.raycluster-worker-svc.test-ns-r8v9f.svc.cluster.local","*-worker-*.test-ns-r8v9f.svc","*-worker-*.test-ns-r8v9f.svc.cluster.local"],"ipAddresses":["10.132.0.30","10.133.0.25","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:43:08.588Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f14408af-0c62-44e9-8f02-aaf094a99a00","rayCluster":"raycluster","podIPs":["10.132.0.30","10.133.0.25"]}
{"level":"info","ts":"2026-09-18T09:43:10.549Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7e99a141-3c5b-4937-aa28-14807acf4cf8"}
{"level":"info","ts":"2026-09-18T09:43:10.549Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7e99a141-3c5b-4937-aa28-14807acf4cf8"}
{"level":"info","ts":"2026-09-18T09:43:10.549Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7e99a141-3c5b-4937-aa28-14807acf4cf8"}
{"level":"info","ts":"2026-09-18T09:43:10.549Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7e99a141-3c5b-4937-aa28-14807acf4cf8","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:43:10.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7e99a141-3c5b-4937-aa28-14807acf4cf8","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:43:10.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7e99a141-3c5b-4937-aa28-14807acf4cf8","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:43:10.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7e99a141-3c5b-4937-aa28-14807acf4cf8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:43:10.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7e99a141-3c5b-4937-aa28-14807acf4cf8","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:43:10.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7e99a141-3c5b-4937-aa28-14807acf4cf8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:43:10.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7e99a141-3c5b-4937-aa28-14807acf4cf8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:43:10.549Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7e99a141-3c5b-4937-aa28-14807acf4cf8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:43:10.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7e99a141-3c5b-4937-aa28-14807acf4cf8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:43:10.549Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7e99a141-3c5b-4937-aa28-14807acf4cf8","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:43:10.549Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7e99a141-3c5b-4937-aa28-14807acf4cf8","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:43:10.549Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7e99a141-3c5b-4937-aa28-14807acf4cf8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:43:10.550Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"7e99a141-3c5b-4937-aa28-14807acf4cf8","seconds":300}
{"level":"info","ts":"2026-09-18T09:43:13.561Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0cfa7ca1-c113-4213-a14e-9d1acdf180cc"}
{"level":"info","ts":"2026-09-18T09:43:13.561Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0cfa7ca1-c113-4213-a14e-9d1acdf180cc"}
{"level":"info","ts":"2026-09-18T09:43:13.561Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0cfa7ca1-c113-4213-a14e-9d1acdf180cc"}
{"level":"info","ts":"2026-09-18T09:43:13.561Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0cfa7ca1-c113-4213-a14e-9d1acdf180cc","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:43:13.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0cfa7ca1-c113-4213-a14e-9d1acdf180cc","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:43:13.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0cfa7ca1-c113-4213-a14e-9d1acdf180cc","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:43:13.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0cfa7ca1-c113-4213-a14e-9d1acdf180cc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:43:13.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0cfa7ca1-c113-4213-a14e-9d1acdf180cc","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:43:13.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0cfa7ca1-c113-4213-a14e-9d1acdf180cc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:43:13.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0cfa7ca1-c113-4213-a14e-9d1acdf180cc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:43:13.561Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0cfa7ca1-c113-4213-a14e-9d1acdf180cc","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:43:13.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0cfa7ca1-c113-4213-a14e-9d1acdf180cc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:43:13.561Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0cfa7ca1-c113-4213-a14e-9d1acdf180cc","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:43:13.561Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0cfa7ca1-c113-4213-a14e-9d1acdf180cc","podName":"raycluster-head-2vjtt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:43:13.561Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0cfa7ca1-c113-4213-a14e-9d1acdf180cc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:43:13.561Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0cfa7ca1-c113-4213-a14e-9d1acdf180cc","seconds":300}
{"level":"info","ts":"2026-09-18T09:43:17.585Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"11f66c71-5d3c-4aa8-93d2-523cb4374c79"}
{"level":"info","ts":"2026-09-18T09:43:17.585Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"11f66c71-5d3c-4aa8-93d2-523cb4374c79"}
{"level":"info","ts":"2026-09-18T09:43:17.585Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"11f66c71-5d3c-4aa8-93d2-523cb4374c79"}
{"level":"info","ts":"2026-09-18T09:43:17.585Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"11f66c71-5d3c-4aa8-93d2-523cb4374c79","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:43:17.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"11f66c71-5d3c-4aa8-93d2-523cb4374c79","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:43:17.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"11f66c71-5d3c-4aa8-93d2-523cb4374c79","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:43:17.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"11f66c71-5d3c-4aa8-93d2-523cb4374c79","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:43:17.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"11f66c71-5d3c-4aa8-93d2-523cb4374c79","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:43:17.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"11f66c71-5d3c-4aa8-93d2-523cb4374c79","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:43:17.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"11f66c71-5d3c-4aa8-93d2-523cb4374c79","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:43:17.585Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"11f66c71-5d3c-4aa8-93d2-523cb4374c79","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:43:17.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"11f66c71-5d3c-4aa8-93d2-523cb4374c79","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:43:17.586Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"11f66c71-5d3c-4aa8-93d2-523cb4374c79","podName":"raycluster-head-2vjtt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:40:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:43:17.586Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"11f66c71-5d3c-4aa8-93d2-523cb4374c79","podName":"raycluster-head-2vjtt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:40:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:43:17.586Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"11f66c71-5d3c-4aa8-93d2-523cb4374c79","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:43:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:43:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:43:17.599Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"23aa206a-0012-4101-b91d-8969c222752b","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:43:17.599Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"79ccc433-4517-4ec9-91de-c8dfb11e03a9","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:43:17.599Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"79ccc433-4517-4ec9-91de-c8dfb11e03a9","pod":"raycluster-head-2vjtt","ip":"10.132.0.30"}
{"level":"info","ts":"2026-09-18T09:43:17.599Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"79ccc433-4517-4ec9-91de-c8dfb11e03a9","pod":"raycluster-small-group-worker-cjnn9","ip":"10.133.0.25"}
{"level":"info","ts":"2026-09-18T09:43:17.599Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"79ccc433-4517-4ec9-91de-c8dfb11e03a9","rayCluster":"raycluster","podIPs":["10.132.0.30","10.133.0.25"]}
{"level":"info","ts":"2026-09-18T09:43:17.609Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"23aa206a-0012-4101-b91d-8969c222752b","name":"raycluster-head"}
{"level":"info","ts":"2026-09-18T09:43:17.622Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"23aa206a-0012-4101-b91d-8969c222752b","name":"raycluster-workers"}
{"level":"info","ts":"2026-09-18T09:43:17.622Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"23aa206a-0012-4101-b91d-8969c222752b","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:43:19.599Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"aa6b5de9-6f0d-463e-aa6e-9508529f3f27"}
{"level":"info","ts":"2026-09-18T09:43:19.599Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"aa6b5de9-6f0d-463e-aa6e-9508529f3f27"}
{"level":"info","ts":"2026-09-18T09:43:19.599Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"aa6b5de9-6f0d-463e-aa6e-9508529f3f27"}
{"level":"info","ts":"2026-09-18T09:43:19.599Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"aa6b5de9-6f0d-463e-aa6e-9508529f3f27","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:43:19.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"aa6b5de9-6f0d-463e-aa6e-9508529f3f27","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:43:19.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"aa6b5de9-6f0d-463e-aa6e-9508529f3f27","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:43:19.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"aa6b5de9-6f0d-463e-aa6e-9508529f3f27","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:43:19.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"aa6b5de9-6f0d-463e-aa6e-9508529f3f27","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:43:19.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"aa6b5de9-6f0d-463e-aa6e-9508529f3f27","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:43:19.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"aa6b5de9-6f0d-463e-aa6e-9508529f3f27","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:43:19.599Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"aa6b5de9-6f0d-463e-aa6e-9508529f3f27","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:43:19.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"aa6b5de9-6f0d-463e-aa6e-9508529f3f27","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:43:19.599Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"aa6b5de9-6f0d-463e-aa6e-9508529f3f27","podName":"raycluster-head-2vjtt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:40:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:43:19.599Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"aa6b5de9-6f0d-463e-aa6e-9508529f3f27","podName":"raycluster-head-2vjtt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:40:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:43:19.599Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"aa6b5de9-6f0d-463e-aa6e-9508529f3f27","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:43:19.599Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"aa6b5de9-6f0d-463e-aa6e-9508529f3f27","seconds":300}
{"level":"info","ts":"2026-09-18T09:43:28.617Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"350224e5-e9e4-4836-9d9f-3d722464df66"}
{"level":"info","ts":"2026-09-18T09:43:28.617Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"350224e5-e9e4-4836-9d9f-3d722464df66"}
{"level":"info","ts":"2026-09-18T09:43:28.617Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"350224e5-e9e4-4836-9d9f-3d722464df66"}
{"level":"info","ts":"2026-09-18T09:43:28.617Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"350224e5-e9e4-4836-9d9f-3d722464df66","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:43:28.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"350224e5-e9e4-4836-9d9f-3d722464df66","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:43:28.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"350224e5-e9e4-4836-9d9f-3d722464df66","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:43:28.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"350224e5-e9e4-4836-9d9f-3d722464df66","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:43:28.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"350224e5-e9e4-4836-9d9f-3d722464df66","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:43:28.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"350224e5-e9e4-4836-9d9f-3d722464df66","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:43:28.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"350224e5-e9e4-4836-9d9f-3d722464df66","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:43:28.617Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"350224e5-e9e4-4836-9d9f-3d722464df66","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:43:28.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"350224e5-e9e4-4836-9d9f-3d722464df66","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:43:28.618Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"350224e5-e9e4-4836-9d9f-3d722464df66","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:43:28.618Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"350224e5-e9e4-4836-9d9f-3d722464df66","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:43:28.618Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"350224e5-e9e4-4836-9d9f-3d722464df66","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:43:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:43:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:43:28.630Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f52fe6fa-d98b-4917-b1bd-d3dfba1aa689","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:43:28.630Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d0b0b6fb-b4f1-4593-9dad-b8cac8d26052","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:43:28.630Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d0b0b6fb-b4f1-4593-9dad-b8cac8d26052","pod":"raycluster-head-2vjtt","ip":"10.132.0.30"}
{"level":"info","ts":"2026-09-18T09:43:28.630Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d0b0b6fb-b4f1-4593-9dad-b8cac8d26052","pod":"raycluster-small-group-worker-cjnn9","ip":"10.133.0.25"}
{"level":"info","ts":"2026-09-18T09:43:28.630Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d0b0b6fb-b4f1-4593-9dad-b8cac8d26052","rayCluster":"raycluster","podIPs":["10.132.0.30","10.133.0.25"]}
{"level":"info","ts":"2026-09-18T09:43:28.643Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f52fe6fa-d98b-4917-b1bd-d3dfba1aa689","name":"raycluster-head"}
{"level":"info","ts":"2026-09-18T09:43:28.654Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f52fe6fa-d98b-4917-b1bd-d3dfba1aa689","name":"raycluster-workers"}
{"level":"info","ts":"2026-09-18T09:43:28.654Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f52fe6fa-d98b-4917-b1bd-d3dfba1aa689","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:43:30.631Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"82b11648-bc61-424b-aac3-f60f499d0ea8"}
{"level":"info","ts":"2026-09-18T09:43:30.631Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"82b11648-bc61-424b-aac3-f60f499d0ea8"}
{"level":"info","ts":"2026-09-18T09:43:30.631Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"82b11648-bc61-424b-aac3-f60f499d0ea8"}
{"level":"info","ts":"2026-09-18T09:43:30.631Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"82b11648-bc61-424b-aac3-f60f499d0ea8","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:43:30.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"82b11648-bc61-424b-aac3-f60f499d0ea8","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:43:30.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"82b11648-bc61-424b-aac3-f60f499d0ea8","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:43:30.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"82b11648-bc61-424b-aac3-f60f499d0ea8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:43:30.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"82b11648-bc61-424b-aac3-f60f499d0ea8","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:43:30.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"82b11648-bc61-424b-aac3-f60f499d0ea8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:43:30.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"82b11648-bc61-424b-aac3-f60f499d0ea8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:43:30.631Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"82b11648-bc61-424b-aac3-f60f499d0ea8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:43:30.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"82b11648-bc61-424b-aac3-f60f499d0ea8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:43:30.631Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"82b11648-bc61-424b-aac3-f60f499d0ea8","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:43:30.631Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"82b11648-bc61-424b-aac3-f60f499d0ea8","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:43:30.632Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"82b11648-bc61-424b-aac3-f60f499d0ea8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:43:30.632Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"82b11648-bc61-424b-aac3-f60f499d0ea8","seconds":300}
{"level":"info","ts":"2026-09-18T09:43:36.801Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d854d386-2e46-4fde-85f5-e6eebc283bed","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:43:36.801Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d854d386-2e46-4fde-85f5-e6eebc283bed","pod":"raycluster-head-2vjtt","ip":"10.132.0.30"}
{"level":"info","ts":"2026-09-18T09:43:36.801Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d854d386-2e46-4fde-85f5-e6eebc283bed","pod":"raycluster-small-group-worker-cjnn9","ip":"10.133.0.25"}
{"level":"info","ts":"2026-09-18T09:43:36.801Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d854d386-2e46-4fde-85f5-e6eebc283bed","rayCluster":"raycluster","podIPs":["10.132.0.30","10.133.0.25"]}
{"level":"info","ts":"2026-09-18T09:44:36.802Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"6f0ab94a-6399-4e29-996d-143a5081f3b0","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:44:36.802Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"6f0ab94a-6399-4e29-996d-143a5081f3b0","pod":"raycluster-head-2vjtt","ip":"10.132.0.30"}
{"level":"info","ts":"2026-09-18T09:44:36.802Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"6f0ab94a-6399-4e29-996d-143a5081f3b0","pod":"raycluster-small-group-worker-cjnn9","ip":"10.133.0.25"}
{"level":"info","ts":"2026-09-18T09:44:36.802Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"6f0ab94a-6399-4e29-996d-143a5081f3b0","rayCluster":"raycluster","podIPs":["10.132.0.30","10.133.0.25"]}
{"level":"info","ts":"2026-09-18T09:44:37.727Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"32a7787c-03b8-41e8-9aa6-345602fa4526"}
{"level":"info","ts":"2026-09-18T09:44:37.727Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"32a7787c-03b8-41e8-9aa6-345602fa4526"}
{"level":"info","ts":"2026-09-18T09:44:37.728Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"32a7787c-03b8-41e8-9aa6-345602fa4526"}
{"level":"info","ts":"2026-09-18T09:44:37.728Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"32a7787c-03b8-41e8-9aa6-345602fa4526","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:44:37.728Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"32a7787c-03b8-41e8-9aa6-345602fa4526","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:44:37.728Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"32a7787c-03b8-41e8-9aa6-345602fa4526","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:44:37.728Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"32a7787c-03b8-41e8-9aa6-345602fa4526","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:44:37.728Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"32a7787c-03b8-41e8-9aa6-345602fa4526","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:44:37.728Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"32a7787c-03b8-41e8-9aa6-345602fa4526","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:44:37.728Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"32a7787c-03b8-41e8-9aa6-345602fa4526","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:44:37.728Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"32a7787c-03b8-41e8-9aa6-345602fa4526","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:44:37.728Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"32a7787c-03b8-41e8-9aa6-345602fa4526","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:44:37.728Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"32a7787c-03b8-41e8-9aa6-345602fa4526","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:44:37.728Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"32a7787c-03b8-41e8-9aa6-345602fa4526","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:44:37.728Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"32a7787c-03b8-41e8-9aa6-345602fa4526","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:44:37.728Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"32a7787c-03b8-41e8-9aa6-345602fa4526","seconds":300}
{"level":"info","ts":"2026-09-18T09:44:38.735Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4c9eb66e-ac55-4802-b362-fe717e5e0da5"}
{"level":"info","ts":"2026-09-18T09:44:38.735Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4c9eb66e-ac55-4802-b362-fe717e5e0da5"}
{"level":"info","ts":"2026-09-18T09:44:38.735Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4c9eb66e-ac55-4802-b362-fe717e5e0da5"}
{"level":"info","ts":"2026-09-18T09:44:38.735Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4c9eb66e-ac55-4802-b362-fe717e5e0da5","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:44:38.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4c9eb66e-ac55-4802-b362-fe717e5e0da5","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:44:38.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4c9eb66e-ac55-4802-b362-fe717e5e0da5","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:44:38.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4c9eb66e-ac55-4802-b362-fe717e5e0da5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:44:38.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4c9eb66e-ac55-4802-b362-fe717e5e0da5","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:44:38.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4c9eb66e-ac55-4802-b362-fe717e5e0da5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:44:38.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4c9eb66e-ac55-4802-b362-fe717e5e0da5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:44:38.735Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4c9eb66e-ac55-4802-b362-fe717e5e0da5","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:44:38.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4c9eb66e-ac55-4802-b362-fe717e5e0da5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:44:38.736Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4c9eb66e-ac55-4802-b362-fe717e5e0da5","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:44:38.736Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4c9eb66e-ac55-4802-b362-fe717e5e0da5","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:44:38.736Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4c9eb66e-ac55-4802-b362-fe717e5e0da5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:44:38.736Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4c9eb66e-ac55-4802-b362-fe717e5e0da5","seconds":300}
{"level":"info","ts":"2026-09-18T09:44:47.771Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4d528ddf-8fb9-43bf-a6cf-f79e03aff860"}
{"level":"info","ts":"2026-09-18T09:44:47.771Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4d528ddf-8fb9-43bf-a6cf-f79e03aff860"}
{"level":"info","ts":"2026-09-18T09:44:47.771Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4d528ddf-8fb9-43bf-a6cf-f79e03aff860"}
{"level":"info","ts":"2026-09-18T09:44:47.771Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4d528ddf-8fb9-43bf-a6cf-f79e03aff860","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:44:47.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4d528ddf-8fb9-43bf-a6cf-f79e03aff860","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:44:47.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4d528ddf-8fb9-43bf-a6cf-f79e03aff860","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:44:47.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4d528ddf-8fb9-43bf-a6cf-f79e03aff860","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:44:47.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4d528ddf-8fb9-43bf-a6cf-f79e03aff860","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:44:47.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4d528ddf-8fb9-43bf-a6cf-f79e03aff860","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:44:47.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4d528ddf-8fb9-43bf-a6cf-f79e03aff860","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:44:47.771Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4d528ddf-8fb9-43bf-a6cf-f79e03aff860","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:44:47.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4d528ddf-8fb9-43bf-a6cf-f79e03aff860","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:44:47.771Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4d528ddf-8fb9-43bf-a6cf-f79e03aff860","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:44:47.771Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4d528ddf-8fb9-43bf-a6cf-f79e03aff860","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:44:47.771Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4d528ddf-8fb9-43bf-a6cf-f79e03aff860","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:44:47.771Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4d528ddf-8fb9-43bf-a6cf-f79e03aff860","seconds":300}
{"level":"info","ts":"2026-09-18T09:44:48.778Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1d5aca1d-631e-4bef-bad2-5a203ac7405a"}
{"level":"info","ts":"2026-09-18T09:44:48.778Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1d5aca1d-631e-4bef-bad2-5a203ac7405a"}
{"level":"info","ts":"2026-09-18T09:44:48.778Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1d5aca1d-631e-4bef-bad2-5a203ac7405a"}
{"level":"info","ts":"2026-09-18T09:44:48.778Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1d5aca1d-631e-4bef-bad2-5a203ac7405a","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:44:48.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1d5aca1d-631e-4bef-bad2-5a203ac7405a","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:44:48.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1d5aca1d-631e-4bef-bad2-5a203ac7405a","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:44:48.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1d5aca1d-631e-4bef-bad2-5a203ac7405a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:44:48.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1d5aca1d-631e-4bef-bad2-5a203ac7405a","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:44:48.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1d5aca1d-631e-4bef-bad2-5a203ac7405a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:44:48.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1d5aca1d-631e-4bef-bad2-5a203ac7405a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:44:48.778Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1d5aca1d-631e-4bef-bad2-5a203ac7405a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:44:48.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1d5aca1d-631e-4bef-bad2-5a203ac7405a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:44:48.778Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1d5aca1d-631e-4bef-bad2-5a203ac7405a","podName":"raycluster-small-group-worker-cjnn9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:40:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:44:48.779Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1d5aca1d-631e-4bef-bad2-5a203ac7405a","podName":"raycluster-small-group-worker-cjnn9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:40:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:44:48.779Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1d5aca1d-631e-4bef-bad2-5a203ac7405a","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:43:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:44:48.792Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"7dc00d78-94eb-4cd6-834f-e69e37a79db8","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:44:48.792Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"5e78ffd7-c9f1-41d4-8227-5e0cba7a79f7","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:44:48.792Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"5e78ffd7-c9f1-41d4-8227-5e0cba7a79f7","pod":"raycluster-head-2vjtt","ip":"10.132.0.30"}
{"level":"info","ts":"2026-09-18T09:44:48.792Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"5e78ffd7-c9f1-41d4-8227-5e0cba7a79f7","pod":"raycluster-small-group-worker-cjnn9","ip":"10.133.0.25"}
{"level":"info","ts":"2026-09-18T09:44:48.792Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"5e78ffd7-c9f1-41d4-8227-5e0cba7a79f7","rayCluster":"raycluster","podIPs":["10.132.0.30","10.133.0.25"]}
{"level":"info","ts":"2026-09-18T09:44:48.813Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"7dc00d78-94eb-4cd6-834f-e69e37a79db8","name":"raycluster-head"}
{"level":"info","ts":"2026-09-18T09:44:48.824Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"7dc00d78-94eb-4cd6-834f-e69e37a79db8","name":"raycluster-workers"}
{"level":"info","ts":"2026-09-18T09:44:48.824Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"7dc00d78-94eb-4cd6-834f-e69e37a79db8","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:44:50.792Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4831f6cd-eee8-4564-963b-b9553d9d8dd9"}
{"level":"info","ts":"2026-09-18T09:44:50.792Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4831f6cd-eee8-4564-963b-b9553d9d8dd9"}
{"level":"info","ts":"2026-09-18T09:44:50.792Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4831f6cd-eee8-4564-963b-b9553d9d8dd9"}
{"level":"info","ts":"2026-09-18T09:44:50.792Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4831f6cd-eee8-4564-963b-b9553d9d8dd9","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:44:50.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4831f6cd-eee8-4564-963b-b9553d9d8dd9","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:44:50.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4831f6cd-eee8-4564-963b-b9553d9d8dd9","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:44:50.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4831f6cd-eee8-4564-963b-b9553d9d8dd9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:44:50.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4831f6cd-eee8-4564-963b-b9553d9d8dd9","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:44:50.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4831f6cd-eee8-4564-963b-b9553d9d8dd9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:44:50.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4831f6cd-eee8-4564-963b-b9553d9d8dd9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:44:50.792Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4831f6cd-eee8-4564-963b-b9553d9d8dd9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:44:50.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4831f6cd-eee8-4564-963b-b9553d9d8dd9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:44:50.792Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4831f6cd-eee8-4564-963b-b9553d9d8dd9","podName":"raycluster-small-group-worker-cjnn9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:40:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:44:50.792Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4831f6cd-eee8-4564-963b-b9553d9d8dd9","podName":"raycluster-small-group-worker-cjnn9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:40:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:44:50.792Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4831f6cd-eee8-4564-963b-b9553d9d8dd9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:44:50.792Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"4831f6cd-eee8-4564-963b-b9553d9d8dd9","seconds":300}
{"level":"info","ts":"2026-09-18T09:44:59.804Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ccb607a6-fae4-417c-9d72-074868259126"}
{"level":"info","ts":"2026-09-18T09:44:59.804Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ccb607a6-fae4-417c-9d72-074868259126"}
{"level":"info","ts":"2026-09-18T09:44:59.804Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ccb607a6-fae4-417c-9d72-074868259126"}
{"level":"info","ts":"2026-09-18T09:44:59.804Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ccb607a6-fae4-417c-9d72-074868259126","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:44:59.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ccb607a6-fae4-417c-9d72-074868259126","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:44:59.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ccb607a6-fae4-417c-9d72-074868259126","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:44:59.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ccb607a6-fae4-417c-9d72-074868259126","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:44:59.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ccb607a6-fae4-417c-9d72-074868259126","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:44:59.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ccb607a6-fae4-417c-9d72-074868259126","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:44:59.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ccb607a6-fae4-417c-9d72-074868259126","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:44:59.804Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ccb607a6-fae4-417c-9d72-074868259126","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:44:59.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ccb607a6-fae4-417c-9d72-074868259126","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:44:59.805Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"ccb607a6-fae4-417c-9d72-074868259126","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:44:59.818Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"fb623b0c-ae1b-4c49-8fd5-419885091582","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:44:59.818Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"394c6408-f715-4247-b2ea-9bb0a0cf94ce","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:44:59.818Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"394c6408-f715-4247-b2ea-9bb0a0cf94ce","pod":"raycluster-head-2vjtt","ip":"10.132.0.30"}
{"level":"info","ts":"2026-09-18T09:44:59.818Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"394c6408-f715-4247-b2ea-9bb0a0cf94ce","pod":"raycluster-small-group-worker-cjnn9","ip":"10.133.0.25"}
{"level":"info","ts":"2026-09-18T09:44:59.818Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"394c6408-f715-4247-b2ea-9bb0a0cf94ce","rayCluster":"raycluster","podIPs":["10.132.0.30","10.133.0.25"]}
{"level":"info","ts":"2026-09-18T09:44:59.829Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"fb623b0c-ae1b-4c49-8fd5-419885091582","name":"raycluster-head"}
{"level":"info","ts":"2026-09-18T09:44:59.841Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"fb623b0c-ae1b-4c49-8fd5-419885091582","name":"raycluster-workers"}
{"level":"info","ts":"2026-09-18T09:44:59.841Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"fb623b0c-ae1b-4c49-8fd5-419885091582","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:44:59.993Z","logger":"controllers.RayJob","msg":"Skipping RayJob managed by a custom controller","RayJob":{"name":"managed-externally","namespace":"test-ns-r8v9f"},"reconcileID":"a622624d-f558-411f-a68e-a7342f19c8a4","managed-by":"kueue.x-k8s.io/multikueue"}
{"level":"info","ts":"2026-09-18T09:45:01.818Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"34e2df26-9324-40c0-a98c-db5e812073fd"}
{"level":"info","ts":"2026-09-18T09:45:01.818Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"34e2df26-9324-40c0-a98c-db5e812073fd"}
{"level":"info","ts":"2026-09-18T09:45:01.818Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"34e2df26-9324-40c0-a98c-db5e812073fd"}
{"level":"info","ts":"2026-09-18T09:45:01.818Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"34e2df26-9324-40c0-a98c-db5e812073fd","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:01.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"34e2df26-9324-40c0-a98c-db5e812073fd","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:01.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"34e2df26-9324-40c0-a98c-db5e812073fd","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:01.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"34e2df26-9324-40c0-a98c-db5e812073fd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:45:01.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"34e2df26-9324-40c0-a98c-db5e812073fd","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:01.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"34e2df26-9324-40c0-a98c-db5e812073fd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:01.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"34e2df26-9324-40c0-a98c-db5e812073fd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:45:01.818Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"34e2df26-9324-40c0-a98c-db5e812073fd","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:45:01.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"34e2df26-9324-40c0-a98c-db5e812073fd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:01.819Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"34e2df26-9324-40c0-a98c-db5e812073fd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:45:01.819Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"34e2df26-9324-40c0-a98c-db5e812073fd","seconds":300}
{"level":"info","ts":"2026-09-18T09:45:03.010Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"0d7fe13a-16c0-43ec-a56a-9ac99a0933d5","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:03.010Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"0d7fe13a-16c0-43ec-a56a-9ac99a0933d5","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:45:03.019Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"0d7fe13a-16c0-43ec-a56a-9ac99a0933d5"}
{"level":"info","ts":"2026-09-18T09:45:03.019Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"0d7fe13a-16c0-43ec-a56a-9ac99a0933d5","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:45:03.019Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"0d7fe13a-16c0-43ec-a56a-9ac99a0933d5","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-sb6s2","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:45:03.019Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"0d7fe13a-16c0-43ec-a56a-9ac99a0933d5","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:45:03.029Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"bcbac32e-9225-4c85-b55b-c4008f94b182","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:03.029Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"bcbac32e-9225-4c85-b55b-c4008f94b182","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:03.029Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"bcbac32e-9225-4c85-b55b-c4008f94b182","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:03.045Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"bcbac32e-9225-4c85-b55b-c4008f94b182","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-09-18T09:45:03.045Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"bcbac32e-9225-4c85-b55b-c4008f94b182","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-sb6s2","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:45:03.046Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"bcbac32e-9225-4c85-b55b-c4008f94b182","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:45:03.057Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"39f18415-2919-4b73-9fcb-1cd17523a8c7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:03.057Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"39f18415-2919-4b73-9fcb-1cd17523a8c7","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:03.057Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"39f18415-2919-4b73-9fcb-1cd17523a8c7","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:03.057Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"39f18415-2919-4b73-9fcb-1cd17523a8c7","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-09-18T09:45:03.057Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"39f18415-2919-4b73-9fcb-1cd17523a8c7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-sb6s2","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:45:03.057Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"39f18415-2919-4b73-9fcb-1cd17523a8c7","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:45:03.062Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"39f18415-2919-4b73-9fcb-1cd17523a8c7","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T09:45:03.062Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"39f18415-2919-4b73-9fcb-1cd17523a8c7"}
{"level":"error","ts":"2026-09-18T09:45:03.062Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"39f18415-2919-4b73-9fcb-1cd17523a8c7","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:03.063Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"48a29b76-f3b5-459d-ba15-bf4027ffae87","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:03.063Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"48a29b76-f3b5-459d-ba15-bf4027ffae87","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:03.063Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"48a29b76-f3b5-459d-ba15-bf4027ffae87","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:03.071Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"48a29b76-f3b5-459d-ba15-bf4027ffae87","JobId":"counter-sb6s2","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:03.071Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"48a29b76-f3b5-459d-ba15-bf4027ffae87"}
{"level":"error","ts":"2026-09-18T09:45:03.071Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"48a29b76-f3b5-459d-ba15-bf4027ffae87","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:03.072Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f5870dea-9e6e-4455-9c92-6dc779feba80","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:03.072Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f5870dea-9e6e-4455-9c92-6dc779feba80","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:03.072Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f5870dea-9e6e-4455-9c92-6dc779feba80","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:03.076Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f5870dea-9e6e-4455-9c92-6dc779feba80","JobId":"counter-sb6s2","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:03.076Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f5870dea-9e6e-4455-9c92-6dc779feba80"}
{"level":"error","ts":"2026-09-18T09:45:03.076Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f5870dea-9e6e-4455-9c92-6dc779feba80","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:03.077Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"68a6f3ec-5dca-4a51-91e0-4c10fe071e07","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:03.077Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"68a6f3ec-5dca-4a51-91e0-4c10fe071e07","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:03.077Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"68a6f3ec-5dca-4a51-91e0-4c10fe071e07","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:03.081Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"68a6f3ec-5dca-4a51-91e0-4c10fe071e07","JobId":"counter-sb6s2","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:03.081Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"68a6f3ec-5dca-4a51-91e0-4c10fe071e07"}
{"level":"error","ts":"2026-09-18T09:45:03.081Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"68a6f3ec-5dca-4a51-91e0-4c10fe071e07","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:03.082Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f694082e-93b1-4803-bfd0-b535c5484da5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:03.082Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f694082e-93b1-4803-bfd0-b535c5484da5","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:03.082Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f694082e-93b1-4803-bfd0-b535c5484da5","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:03.087Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f694082e-93b1-4803-bfd0-b535c5484da5","JobId":"counter-sb6s2","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:03.087Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f694082e-93b1-4803-bfd0-b535c5484da5"}
{"level":"error","ts":"2026-09-18T09:45:03.087Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f694082e-93b1-4803-bfd0-b535c5484da5","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:03.168Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f34cee09-be8e-440f-a847-71bdb0e501d9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:03.168Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f34cee09-be8e-440f-a847-71bdb0e501d9","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:03.168Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f34cee09-be8e-440f-a847-71bdb0e501d9","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:03.173Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f34cee09-be8e-440f-a847-71bdb0e501d9","JobId":"counter-sb6s2","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:03.174Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f34cee09-be8e-440f-a847-71bdb0e501d9"}
{"level":"error","ts":"2026-09-18T09:45:03.174Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f34cee09-be8e-440f-a847-71bdb0e501d9","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:03.334Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"aeeb9e91-cd11-4721-8ba4-63cd3dbcff7c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:03.334Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"aeeb9e91-cd11-4721-8ba4-63cd3dbcff7c","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:03.334Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"aeeb9e91-cd11-4721-8ba4-63cd3dbcff7c","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:03.340Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"aeeb9e91-cd11-4721-8ba4-63cd3dbcff7c","JobId":"counter-sb6s2","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:03.340Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"aeeb9e91-cd11-4721-8ba4-63cd3dbcff7c"}
{"level":"error","ts":"2026-09-18T09:45:03.340Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"aeeb9e91-cd11-4721-8ba4-63cd3dbcff7c","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:03.660Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"c0c9f20c-aceb-487b-8ff1-3757654dc8ae","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:03.660Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"c0c9f20c-aceb-487b-8ff1-3757654dc8ae","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:03.660Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"c0c9f20c-aceb-487b-8ff1-3757654dc8ae","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:03.666Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"c0c9f20c-aceb-487b-8ff1-3757654dc8ae","JobId":"counter-sb6s2","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:03.666Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"c0c9f20c-aceb-487b-8ff1-3757654dc8ae"}
{"level":"error","ts":"2026-09-18T09:45:03.666Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"c0c9f20c-aceb-487b-8ff1-3757654dc8ae","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:04.079Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"dbb77e0f-ed1b-4952-adad-6de52f4a11c0","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:04.079Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"dbb77e0f-ed1b-4952-adad-6de52f4a11c0","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:04.079Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"dbb77e0f-ed1b-4952-adad-6de52f4a11c0","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:04.085Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"dbb77e0f-ed1b-4952-adad-6de52f4a11c0","JobId":"counter-sb6s2","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:04.085Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"dbb77e0f-ed1b-4952-adad-6de52f4a11c0"}
{"level":"error","ts":"2026-09-18T09:45:04.085Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"dbb77e0f-ed1b-4952-adad-6de52f4a11c0","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:04.307Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"5c6219f2-4ff8-4175-8d9e-4891eb5788ab","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:04.307Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"5c6219f2-4ff8-4175-8d9e-4891eb5788ab","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:04.307Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"5c6219f2-4ff8-4175-8d9e-4891eb5788ab","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:04.312Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"5c6219f2-4ff8-4175-8d9e-4891eb5788ab","JobId":"counter-sb6s2","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:04.312Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"5c6219f2-4ff8-4175-8d9e-4891eb5788ab"}
{"level":"error","ts":"2026-09-18T09:45:04.312Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"5c6219f2-4ff8-4175-8d9e-4891eb5788ab","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:06.872Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"613e4404-49ee-4561-9c28-5a689860891f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:06.872Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"613e4404-49ee-4561-9c28-5a689860891f","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:06.873Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"613e4404-49ee-4561-9c28-5a689860891f","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:06.878Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"613e4404-49ee-4561-9c28-5a689860891f","JobId":"counter-sb6s2","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:06.878Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"613e4404-49ee-4561-9c28-5a689860891f"}
{"level":"error","ts":"2026-09-18T09:45:06.878Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"613e4404-49ee-4561-9c28-5a689860891f","error":"Job counter-sb6s2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:11.999Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"b617660b-72f2-4b63-b3b4-2304ccfc6942","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:11.999Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"b617660b-72f2-4b63-b3b4-2304ccfc6942","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:11.999Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"b617660b-72f2-4b63-b3b4-2304ccfc6942","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:12.005Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"b617660b-72f2-4b63-b3b4-2304ccfc6942","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:12.005Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"b617660b-72f2-4b63-b3b4-2304ccfc6942","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:45:12.017Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"86ba9ce8-bca0-4889-b91b-540982b193e7","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:12.017Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"86ba9ce8-bca0-4889-b91b-540982b193e7","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:12.017Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"86ba9ce8-bca0-4889-b91b-540982b193e7","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:12.019Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"86ba9ce8-bca0-4889-b91b-540982b193e7","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:15.018Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"805a5fb7-d21a-46d7-8b64-fad89260540f","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:15.018Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"805a5fb7-d21a-46d7-8b64-fad89260540f","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:15.018Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"805a5fb7-d21a-46d7-8b64-fad89260540f","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:15.021Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"805a5fb7-d21a-46d7-8b64-fad89260540f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:15.021Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"805a5fb7-d21a-46d7-8b64-fad89260540f","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:45:15.035Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"3553d04a-76cc-4c3f-99d7-812146110fc9","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:15.035Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"3553d04a-76cc-4c3f-99d7-812146110fc9","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:15.035Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"3553d04a-76cc-4c3f-99d7-812146110fc9","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:15.037Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"3553d04a-76cc-4c3f-99d7-812146110fc9","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:18.036Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"e4889fe9-e913-426f-94df-25cecc3bc60d","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:18.036Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"e4889fe9-e913-426f-94df-25cecc3bc60d","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:18.036Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"e4889fe9-e913-426f-94df-25cecc3bc60d","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:18.039Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"e4889fe9-e913-426f-94df-25cecc3bc60d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z","endTime":"2026-09-18T09:45:15Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:18.039Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"e4889fe9-e913-426f-94df-25cecc3bc60d","old JobStatus":"RUNNING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:45:18.055Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"cf81bdc9-90a7-49e9-aca7-9e06b789b4e8","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:18.055Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"cf81bdc9-90a7-49e9-aca7-9e06b789b4e8","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:18.055Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"cf81bdc9-90a7-49e9-aca7-9e06b789b4e8","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:18.058Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"cf81bdc9-90a7-49e9-aca7-9e06b789b4e8","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z","endTime":"2026-09-18T09:45:15Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z","endTime":"2026-09-18T09:45:15Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:18.109Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"9865e36e-ae87-41e9-b59a-df900a86fb2c","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:18.109Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"9865e36e-ae87-41e9-b59a-df900a86fb2c","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:45:18.117Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"9865e36e-ae87-41e9-b59a-df900a86fb2c"}
{"level":"info","ts":"2026-09-18T09:45:18.117Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"9865e36e-ae87-41e9-b59a-df900a86fb2c","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:45:18.117Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"9865e36e-ae87-41e9-b59a-df900a86fb2c","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-mvhhs","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:45:18.117Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"9865e36e-ae87-41e9-b59a-df900a86fb2c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:45:18.125Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"6d4175a7-bf25-4ef0-a44c-82ab617f5d55","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:18.125Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"6d4175a7-bf25-4ef0-a44c-82ab617f5d55","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:18.125Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"6d4175a7-bf25-4ef0-a44c-82ab617f5d55","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:18.134Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"6d4175a7-bf25-4ef0-a44c-82ab617f5d55","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-09-18T09:45:18.134Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"6d4175a7-bf25-4ef0-a44c-82ab617f5d55","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-mvhhs","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-mvhhs","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:45:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:45:18.134Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"6d4175a7-bf25-4ef0-a44c-82ab617f5d55","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:45:18.144Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"55b48a75-e823-41c2-815a-1e743dfc9b63","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:18.145Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"55b48a75-e823-41c2-815a-1e743dfc9b63","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:18.145Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"55b48a75-e823-41c2-815a-1e743dfc9b63","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:18.145Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"55b48a75-e823-41c2-815a-1e743dfc9b63","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-09-18T09:45:18.145Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"55b48a75-e823-41c2-815a-1e743dfc9b63","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-mvhhs","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-mvhhs","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:45:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:45:18.145Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"55b48a75-e823-41c2-815a-1e743dfc9b63","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:45:18.150Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"55b48a75-e823-41c2-815a-1e743dfc9b63","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T09:45:18.150Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"55b48a75-e823-41c2-815a-1e743dfc9b63"}
{"level":"error","ts":"2026-09-18T09:45:18.151Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"55b48a75-e823-41c2-815a-1e743dfc9b63","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:18.151Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"2595483c-3c60-4841-a925-66429a720068","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:18.151Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"2595483c-3c60-4841-a925-66429a720068","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:18.151Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"2595483c-3c60-4841-a925-66429a720068","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:18.153Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"2595483c-3c60-4841-a925-66429a720068","JobId":"fail-mvhhs","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:18.153Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"2595483c-3c60-4841-a925-66429a720068"}
{"level":"error","ts":"2026-09-18T09:45:18.153Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"2595483c-3c60-4841-a925-66429a720068","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:18.155Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"5c1972b9-b4ac-496f-b12b-6e378d368b27","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:18.155Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"5c1972b9-b4ac-496f-b12b-6e378d368b27","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:18.155Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"5c1972b9-b4ac-496f-b12b-6e378d368b27","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:18.160Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"5c1972b9-b4ac-496f-b12b-6e378d368b27","JobId":"fail-mvhhs","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:18.160Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"5c1972b9-b4ac-496f-b12b-6e378d368b27"}
{"level":"error","ts":"2026-09-18T09:45:18.160Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"5c1972b9-b4ac-496f-b12b-6e378d368b27","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:18.160Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"e64182c5-06c9-4e17-9186-04b8ba4bdc0a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:18.160Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"e64182c5-06c9-4e17-9186-04b8ba4bdc0a","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:18.160Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"e64182c5-06c9-4e17-9186-04b8ba4bdc0a","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:18.165Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"e64182c5-06c9-4e17-9186-04b8ba4bdc0a","JobId":"fail-mvhhs","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:18.165Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"e64182c5-06c9-4e17-9186-04b8ba4bdc0a"}
{"level":"error","ts":"2026-09-18T09:45:18.165Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"e64182c5-06c9-4e17-9186-04b8ba4bdc0a","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:18.182Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"a15fcd7f-ab9f-4f78-9c35-406829128a1a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:18.182Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"a15fcd7f-ab9f-4f78-9c35-406829128a1a","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:18.182Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"a15fcd7f-ab9f-4f78-9c35-406829128a1a","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:18.186Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"a15fcd7f-ab9f-4f78-9c35-406829128a1a","JobId":"fail-mvhhs","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:18.186Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"a15fcd7f-ab9f-4f78-9c35-406829128a1a"}
{"level":"error","ts":"2026-09-18T09:45:18.186Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"a15fcd7f-ab9f-4f78-9c35-406829128a1a","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:18.267Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"023a3ce7-d975-4e0c-9fbb-6ecf478a1334","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:18.267Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"023a3ce7-d975-4e0c-9fbb-6ecf478a1334","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:18.267Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"023a3ce7-d975-4e0c-9fbb-6ecf478a1334","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:18.271Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"023a3ce7-d975-4e0c-9fbb-6ecf478a1334","JobId":"fail-mvhhs","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:18.271Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"023a3ce7-d975-4e0c-9fbb-6ecf478a1334"}
{"level":"error","ts":"2026-09-18T09:45:18.271Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"023a3ce7-d975-4e0c-9fbb-6ecf478a1334","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:18.432Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"0aed7011-2796-4211-8a1c-ad58cff8ea2f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:18.432Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"0aed7011-2796-4211-8a1c-ad58cff8ea2f","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:18.432Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"0aed7011-2796-4211-8a1c-ad58cff8ea2f","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:18.436Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"0aed7011-2796-4211-8a1c-ad58cff8ea2f","JobId":"fail-mvhhs","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:18.437Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"0aed7011-2796-4211-8a1c-ad58cff8ea2f"}
{"level":"error","ts":"2026-09-18T09:45:18.437Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"0aed7011-2796-4211-8a1c-ad58cff8ea2f","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:18.757Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"127da824-474e-4b92-8302-a525b945a085","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:18.757Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"127da824-474e-4b92-8302-a525b945a085","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:18.757Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"127da824-474e-4b92-8302-a525b945a085","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:18.763Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"127da824-474e-4b92-8302-a525b945a085","JobId":"fail-mvhhs","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:18.763Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"127da824-474e-4b92-8302-a525b945a085"}
{"level":"error","ts":"2026-09-18T09:45:18.763Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"127da824-474e-4b92-8302-a525b945a085","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:19.156Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"30d68cea-db6f-4df6-86f2-d628446cb7f7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:19.156Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"30d68cea-db6f-4df6-86f2-d628446cb7f7","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:19.156Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"30d68cea-db6f-4df6-86f2-d628446cb7f7","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:19.161Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"30d68cea-db6f-4df6-86f2-d628446cb7f7","JobId":"fail-mvhhs","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:19.161Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"30d68cea-db6f-4df6-86f2-d628446cb7f7"}
{"level":"error","ts":"2026-09-18T09:45:19.161Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"30d68cea-db6f-4df6-86f2-d628446cb7f7","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:19.404Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"040190de-0827-46d5-84b3-94015f0426f2","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:19.404Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"040190de-0827-46d5-84b3-94015f0426f2","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:19.404Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"040190de-0827-46d5-84b3-94015f0426f2","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:19.409Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"040190de-0827-46d5-84b3-94015f0426f2","JobId":"fail-mvhhs","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:19.409Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"040190de-0827-46d5-84b3-94015f0426f2"}
{"level":"error","ts":"2026-09-18T09:45:19.409Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"040190de-0827-46d5-84b3-94015f0426f2","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:21.056Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"bc8c0576-c78f-44c3-91e3-f8cffd6df763","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:21.056Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"bc8c0576-c78f-44c3-91e3-f8cffd6df763","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:21.056Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"bc8c0576-c78f-44c3-91e3-f8cffd6df763","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:21.061Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"bc8c0576-c78f-44c3-91e3-f8cffd6df763","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z","endTime":"2026-09-18T09:45:15Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z","endTime":"2026-09-18T09:45:15Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:21.920Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"c8985509-ca70-4b19-8c20-9f64fc6cb8d2","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:21.920Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"c8985509-ca70-4b19-8c20-9f64fc6cb8d2","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:21.920Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"c8985509-ca70-4b19-8c20-9f64fc6cb8d2","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:21.923Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"c8985509-ca70-4b19-8c20-9f64fc6cb8d2","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z","endTime":"2026-09-18T09:45:15Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z","endTime":"2026-09-18T09:45:15Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:21.970Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"963aa040-aaeb-465b-906e-db6031685eb9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:21.970Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"963aa040-aaeb-465b-906e-db6031685eb9","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:21.970Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"963aa040-aaeb-465b-906e-db6031685eb9","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:21.973Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"963aa040-aaeb-465b-906e-db6031685eb9","JobId":"fail-mvhhs","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:21.973Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"963aa040-aaeb-465b-906e-db6031685eb9"}
{"level":"error","ts":"2026-09-18T09:45:21.973Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"963aa040-aaeb-465b-906e-db6031685eb9","error":"Job fail-mvhhs does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:22.928Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"53a4a210-622c-4cfe-9674-84d974474775","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:22.928Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"53a4a210-622c-4cfe-9674-84d974474775","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:22.928Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"53a4a210-622c-4cfe-9674-84d974474775","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:22.934Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"53a4a210-622c-4cfe-9674-84d974474775","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z","endTime":"2026-09-18T09:45:15Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z","endTime":"2026-09-18T09:45:15Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:22.947Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f706d304-c879-4681-a0e0-d8420c6ad667","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:22.947Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f706d304-c879-4681-a0e0-d8420c6ad667","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:22.947Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f706d304-c879-4681-a0e0-d8420c6ad667","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:22.949Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f706d304-c879-4681-a0e0-d8420c6ad667","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z","endTime":"2026-09-18T09:45:15Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:45:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:11Z","endTime":"2026-09-18T09:45:15Z"},"jobId":"counter-sb6s2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-09-18T09:45:03Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:22.949Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"f706d304-c879-4681-a0e0-d8420c6ad667","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-09-18T09:45:22.962Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"9fbd0d5e-12d1-493c-a851-614b6b24e280","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:22.963Z","logger":"controllers.RayJob","msg":"RayJob is using an existing RayCluster via clusterSelector; skipping resource deletion.","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"9fbd0d5e-12d1-493c-a851-614b6b24e280","RayClusterSelector":{"ray.io/cluster":"raycluster"}}
{"level":"info","ts":"2026-09-18T09:45:24.062Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"ffc58520-e970-4a9a-808d-1b066fb19520","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:24.062Z","logger":"controllers.RayJob","msg":"RayJob is using an existing RayCluster via clusterSelector; skipping resource deletion.","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"ffc58520-e970-4a9a-808d-1b066fb19520","RayClusterSelector":{"ray.io/cluster":"raycluster"}}
{"level":"info","ts":"2026-09-18T09:45:27.094Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"fa4b453b-36cb-48a3-97e5-e461e8d65576","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:27.094Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"fa4b453b-36cb-48a3-97e5-e461e8d65576","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:27.094Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"fa4b453b-36cb-48a3-97e5-e461e8d65576","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:27.097Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"fa4b453b-36cb-48a3-97e5-e461e8d65576","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-mvhhs","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:45:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:25Z","endTime":"2026-09-18T09:45:27Z"},"jobId":"fail-mvhhs","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:45:25,700\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mvhhs: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:45:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:27.097Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"fa4b453b-36cb-48a3-97e5-e461e8d65576","old JobStatus":"","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:45:27.110Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"f42e2cdf-2074-4ad1-99ed-b7c35772b321","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:27.110Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"f42e2cdf-2074-4ad1-99ed-b7c35772b321","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:27.110Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"f42e2cdf-2074-4ad1-99ed-b7c35772b321","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:27.113Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"f42e2cdf-2074-4ad1-99ed-b7c35772b321","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:25Z","endTime":"2026-09-18T09:45:27Z"},"jobId":"fail-mvhhs","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:45:25,700\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mvhhs: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:45:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:25Z","endTime":"2026-09-18T09:45:27Z"},"jobId":"fail-mvhhs","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:45:25,700\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mvhhs: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:45:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:27.313Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"515fafe1-761b-4e31-9fc7-3f0a90d03591","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:27.313Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"515fafe1-761b-4e31-9fc7-3f0a90d03591","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:45:27.326Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"515fafe1-761b-4e31-9fc7-3f0a90d03591"}
{"level":"info","ts":"2026-09-18T09:45:27.326Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"515fafe1-761b-4e31-9fc7-3f0a90d03591","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:45:27.326Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"515fafe1-761b-4e31-9fc7-3f0a90d03591","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:45:27.326Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"515fafe1-761b-4e31-9fc7-3f0a90d03591","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:45:27.339Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"0e104b18-2791-46ff-99e5-1221158bab60","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:27.339Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"0e104b18-2791-46ff-99e5-1221158bab60"}
{"level":"info","ts":"2026-09-18T09:45:27.339Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"0e104b18-2791-46ff-99e5-1221158bab60","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:45:27.339Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"0e104b18-2791-46ff-99e5-1221158bab60","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-zgx5p","rayClusterName":"counter-v2px5","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:45:27.339Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"0e104b18-2791-46ff-99e5-1221158bab60","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:45:27.346Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"0e104b18-2791-46ff-99e5-1221158bab60","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T09:45:27.346Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"0e104b18-2791-46ff-99e5-1221158bab60"}
{"level":"error","ts":"2026-09-18T09:45:27.346Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"0e104b18-2791-46ff-99e5-1221158bab60","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:27.346Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b688f668-14c7-45e0-903b-651d37a94feb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:27.346Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b688f668-14c7-45e0-903b-651d37a94feb","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.346Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b688f668-14c7-45e0-903b-651d37a94feb","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:27.346Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b688f668-14c7-45e0-903b-651d37a94feb","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:27.365Z","logger":"raycluster-resource","msg":"default","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.365Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-wmhkk","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:45:27.376Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b688f668-14c7-45e0-903b-651d37a94feb","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:27.376Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"4386fb9e-df93-428e-9388-3e665332b560","rayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.376Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"4386fb9e-df93-428e-9388-3e665332b560","rayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.376Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b688f668-14c7-45e0-903b-651d37a94feb"}
{"level":"info","ts":"2026-09-18T09:45:27.376Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b688f668-14c7-45e0-903b-651d37a94feb","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:27.376Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b688f668-14c7-45e0-903b-651d37a94feb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:45:27.376Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"cc76907f-a89a-44dd-b8d4-8898cc13d6f1","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.377Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c20fa14e-1a05-46ac-9294-c2c6a9623e1f","namespacedName":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:27.377Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c20fa14e-1a05-46ac-9294-c2c6a9623e1f","mode":"IntegratedOAuth","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.377Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c20fa14e-1a05-46ac-9294-c2c6a9623e1f","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.377Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c20fa14e-1a05-46ac-9294-c2c6a9623e1f","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.377Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"9de64ef8-b279-4831-acd8-a57279138726"}
{"level":"info","ts":"2026-09-18T09:45:27.377Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"9de64ef8-b279-4831-acd8-a57279138726"}
{"level":"info","ts":"2026-09-18T09:45:27.377Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"9de64ef8-b279-4831-acd8-a57279138726"}
{"level":"info","ts":"2026-09-18T09:45:27.376Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"fd4d79dc-b438-488b-92c7-1aac5f8e468c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:27.377Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"fd4d79dc-b438-488b-92c7-1aac5f8e468c","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.377Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"fd4d79dc-b438-488b-92c7-1aac5f8e468c","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:27.377Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"fd4d79dc-b438-488b-92c7-1aac5f8e468c"}
{"level":"info","ts":"2026-09-18T09:45:27.377Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"fd4d79dc-b438-488b-92c7-1aac5f8e468c","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:27.377Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"fd4d79dc-b438-488b-92c7-1aac5f8e468c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:45:27.383Z","logger":"raycluster-resource","msg":"default","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.383Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-wmhkk","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:45:27.387Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"9de64ef8-b279-4831-acd8-a57279138726","name":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:27.387Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"9de64ef8-b279-4831-acd8-a57279138726","error":"head certificate secret ray-head-secret-counter-wmhkk not found"}
{"level":"error","ts":"2026-09-18T09:45:27.387Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"9de64ef8-b279-4831-acd8-a57279138726","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:27.387Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"9de64ef8-b279-4831-acd8-a57279138726","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found"}
{"level":"info","ts":"2026-09-18T09:45:27.387Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"9de64ef8-b279-4831-acd8-a57279138726","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.387Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"9de64ef8-b279-4831-acd8-a57279138726","serviceSelectors":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/identifier":"counter-wmhkk-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.387Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"9de64ef8-b279-4831-acd8-a57279138726","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.387Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"9de64ef8-b279-4831-acd8-a57279138726","error":"unable to find head service. cluster name counter-wmhkk, filter labels [test-ns-q2lj2 map[ray.io/cluster:counter-wmhkk ray.io/identifier:counter-wmhkk-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-09-18T09:45:27.387Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"9de64ef8-b279-4831-acd8-a57279138726"}
{"level":"error","ts":"2026-09-18T09:45:27.388Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"9de64ef8-b279-4831-acd8-a57279138726","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:27.466Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3624a917-ac29-4eee-bba1-8cddc161270d"}
{"level":"info","ts":"2026-09-18T09:45:27.466Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3624a917-ac29-4eee-bba1-8cddc161270d"}
{"level":"info","ts":"2026-09-18T09:45:27.466Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3624a917-ac29-4eee-bba1-8cddc161270d"}
{"level":"info","ts":"2026-09-18T09:45:27.466Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"4386fb9e-df93-428e-9388-3e665332b560","rayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.466Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"4386fb9e-df93-428e-9388-3e665332b560","rayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.467Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"78871799-f69b-4f24-a19f-c3f97bdc3426","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:27.467Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"78871799-f69b-4f24-a19f-c3f97bdc3426","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.467Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"78871799-f69b-4f24-a19f-c3f97bdc3426","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:27.467Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"78871799-f69b-4f24-a19f-c3f97bdc3426"}
{"level":"info","ts":"2026-09-18T09:45:27.467Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"78871799-f69b-4f24-a19f-c3f97bdc3426","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:27.467Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"78871799-f69b-4f24-a19f-c3f97bdc3426","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:45:27.467Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"cc76907f-a89a-44dd-b8d4-8898cc13d6f1","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.467Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"d24d5fef-d830-410f-9370-d6c175cec4ef","cluster":"counter-wmhkk"}
{"level":"error","ts":"2026-09-18T09:45:27.476Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3624a917-ac29-4eee-bba1-8cddc161270d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcileHeadService-fm","error":"services \"counter-wmhkk-head-svc\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:27.476Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3624a917-ac29-4eee-bba1-8cddc161270d","error":"services \"counter-wmhkk-head-svc\" already exists"}
{"level":"info","ts":"2026-09-18T09:45:27.476Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3624a917-ac29-4eee-bba1-8cddc161270d","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.476Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3624a917-ac29-4eee-bba1-8cddc161270d","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.476Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3624a917-ac29-4eee-bba1-8cddc161270d","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.476Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3624a917-ac29-4eee-bba1-8cddc161270d","name":"counter-wmhkk","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:27.476Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c20fa14e-1a05-46ac-9294-c2c6a9623e1f","name":"counter-wmhkk-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:45:27.480Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"4386fb9e-df93-428e-9388-3e665332b560","rayCluster":"counter-wmhkk","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T09:45:27.480Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"4386fb9e-df93-428e-9388-3e665332b560","rayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.485Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"d24d5fef-d830-410f-9370-d6c175cec4ef","name":"counter-wmhkk-head"}
{"level":"info","ts":"2026-09-18T09:45:27.485Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c20fa14e-1a05-46ac-9294-c2c6a9623e1f","name":"kuberay-gateway-access","namespace":"test-ns-q2lj2","operation":"created"}
{"level":"info","ts":"2026-09-18T09:45:27.485Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3624a917-ac29-4eee-bba1-8cddc161270d","name":"counter-wmhkk","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-wmhkk\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-q2lj2","name":"counter-wmhkk"}}
{"level":"info","ts":"2026-09-18T09:45:27.485Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3624a917-ac29-4eee-bba1-8cddc161270d"}
{"level":"error","ts":"2026-09-18T09:45:27.485Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3624a917-ac29-4eee-bba1-8cddc161270d","error":"services \"counter-wmhkk-head-svc\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:27.485Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"82e4b996-fbf1-4284-a7de-33f87670cc9c"}
{"level":"info","ts":"2026-09-18T09:45:27.485Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"82e4b996-fbf1-4284-a7de-33f87670cc9c"}
{"level":"info","ts":"2026-09-18T09:45:27.485Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"82e4b996-fbf1-4284-a7de-33f87670cc9c"}
{"level":"info","ts":"2026-09-18T09:45:27.485Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"82e4b996-fbf1-4284-a7de-33f87670cc9c","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:27.486Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"82e4b996-fbf1-4284-a7de-33f87670cc9c","error":"head certificate secret ray-head-secret-counter-wmhkk not found"}
{"level":"error","ts":"2026-09-18T09:45:27.486Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"82e4b996-fbf1-4284-a7de-33f87670cc9c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:27.486Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"82e4b996-fbf1-4284-a7de-33f87670cc9c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found"}
{"level":"info","ts":"2026-09-18T09:45:27.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"82e4b996-fbf1-4284-a7de-33f87670cc9c","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"82e4b996-fbf1-4284-a7de-33f87670cc9c","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"82e4b996-fbf1-4284-a7de-33f87670cc9c","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.486Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"82e4b996-fbf1-4284-a7de-33f87670cc9c","name":"counter-wmhkk","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:27.490Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"4386fb9e-df93-428e-9388-3e665332b560","rayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.499Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c20fa14e-1a05-46ac-9294-c2c6a9623e1f","name":"test-ns-q2lj2-counter-wmhkk","operation":"created"}
{"level":"info","ts":"2026-09-18T09:45:27.566Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"82e4b996-fbf1-4284-a7de-33f87670cc9c"}
{"level":"error","ts":"2026-09-18T09:45:27.566Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"82e4b996-fbf1-4284-a7de-33f87670cc9c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:27.566Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"5289ea01-60d9-42b2-a925-e556c6a27713"}
{"level":"info","ts":"2026-09-18T09:45:27.566Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"5289ea01-60d9-42b2-a925-e556c6a27713"}
{"level":"info","ts":"2026-09-18T09:45:27.566Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"5289ea01-60d9-42b2-a925-e556c6a27713"}
{"level":"info","ts":"2026-09-18T09:45:27.566Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"5289ea01-60d9-42b2-a925-e556c6a27713","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:27.566Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"5289ea01-60d9-42b2-a925-e556c6a27713","error":"head certificate secret ray-head-secret-counter-wmhkk not found"}
{"level":"error","ts":"2026-09-18T09:45:27.566Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"5289ea01-60d9-42b2-a925-e556c6a27713","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:27.566Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"5289ea01-60d9-42b2-a925-e556c6a27713","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found"}
{"level":"info","ts":"2026-09-18T09:45:27.566Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"5289ea01-60d9-42b2-a925-e556c6a27713","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.566Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"5289ea01-60d9-42b2-a925-e556c6a27713","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.566Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"5289ea01-60d9-42b2-a925-e556c6a27713","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.566Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"5289ea01-60d9-42b2-a925-e556c6a27713","name":"counter-wmhkk","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:27.567Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"d24d5fef-d830-410f-9370-d6c175cec4ef","name":"counter-wmhkk-workers"}
{"level":"info","ts":"2026-09-18T09:45:27.567Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"d24d5fef-d830-410f-9370-d6c175cec4ef","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.567Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"224216da-cb40-476c-8e7a-d2c914135a35","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.567Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c20fa14e-1a05-46ac-9294-c2c6a9623e1f","name":"kube-rbac-proxy-config-counter-wmhkk","operation":"created"}
{"level":"info","ts":"2026-09-18T09:45:27.568Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"f317bfb9-4be5-421e-9630-0de7e40e697d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:27.568Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"f317bfb9-4be5-421e-9630-0de7e40e697d","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.568Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"f317bfb9-4be5-421e-9630-0de7e40e697d","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:27.568Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"f317bfb9-4be5-421e-9630-0de7e40e697d"}
{"level":"info","ts":"2026-09-18T09:45:27.568Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"f317bfb9-4be5-421e-9630-0de7e40e697d","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:27.568Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"f317bfb9-4be5-421e-9630-0de7e40e697d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:27.568Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"f317bfb9-4be5-421e-9630-0de7e40e697d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:45:27.568Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"4386fb9e-df93-428e-9388-3e665332b560","rayCluster":"counter-wmhkk","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:45:27.574Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"5289ea01-60d9-42b2-a925-e556c6a27713","name":"counter-wmhkk","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-wmhkk\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-q2lj2","name":"counter-wmhkk"}}
{"level":"info","ts":"2026-09-18T09:45:27.574Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"5289ea01-60d9-42b2-a925-e556c6a27713"}
{"level":"error","ts":"2026-09-18T09:45:27.574Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"5289ea01-60d9-42b2-a925-e556c6a27713","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:45:27.575Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c20fa14e-1a05-46ac-9294-c2c6a9623e1f","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-wmhkk\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:27.575Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c20fa14e-1a05-46ac-9294-c2c6a9623e1f"}
{"level":"error","ts":"2026-09-18T09:45:27.575Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c20fa14e-1a05-46ac-9294-c2c6a9623e1f","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-wmhkk\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:27.575Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"7e36f4ad-1d6a-4e7e-9bc8-8df4888e0f1b","namespacedName":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:27.575Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"7e36f4ad-1d6a-4e7e-9bc8-8df4888e0f1b","mode":"IntegratedOAuth","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.575Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"7e36f4ad-1d6a-4e7e-9bc8-8df4888e0f1b","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.578Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"224216da-cb40-476c-8e7a-d2c914135a35","name":"counter-wmhkk-head"}
{"level":"info","ts":"2026-09-18T09:45:27.579Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"4386fb9e-df93-428e-9388-3e665332b560","rayCluster":"counter-wmhkk","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:45:27.580Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"4386fb9e-df93-428e-9388-3e665332b560"}
{"level":"info","ts":"2026-09-18T09:45:27.580Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"450599f4-b504-45c7-8965-edd4914d64c0","rayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.580Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"450599f4-b504-45c7-8965-edd4914d64c0"}
{"level":"info","ts":"2026-09-18T09:45:27.580Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"28c5b7f0-a9f8-43ea-8f6e-d9b77e244363","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:27.580Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"28c5b7f0-a9f8-43ea-8f6e-d9b77e244363","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.580Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"28c5b7f0-a9f8-43ea-8f6e-d9b77e244363","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:27.580Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"28c5b7f0-a9f8-43ea-8f6e-d9b77e244363"}
{"level":"info","ts":"2026-09-18T09:45:27.580Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"28c5b7f0-a9f8-43ea-8f6e-d9b77e244363","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:27.580Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"28c5b7f0-a9f8-43ea-8f6e-d9b77e244363","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:27.587Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"4e732748-e617-4213-b19a-1f4381b29f42"}
{"level":"info","ts":"2026-09-18T09:45:27.587Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"4e732748-e617-4213-b19a-1f4381b29f42"}
{"level":"info","ts":"2026-09-18T09:45:27.587Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"4e732748-e617-4213-b19a-1f4381b29f42"}
{"level":"info","ts":"2026-09-18T09:45:27.587Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"4e732748-e617-4213-b19a-1f4381b29f42","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:27.587Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"4e732748-e617-4213-b19a-1f4381b29f42","error":"head certificate secret ray-head-secret-counter-wmhkk not found"}
{"level":"error","ts":"2026-09-18T09:45:27.587Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"4e732748-e617-4213-b19a-1f4381b29f42","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:27.587Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"4e732748-e617-4213-b19a-1f4381b29f42","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found"}
{"level":"info","ts":"2026-09-18T09:45:27.587Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"4e732748-e617-4213-b19a-1f4381b29f42","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.587Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"4e732748-e617-4213-b19a-1f4381b29f42","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.587Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"4e732748-e617-4213-b19a-1f4381b29f42","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.587Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"4e732748-e617-4213-b19a-1f4381b29f42"}
{"level":"error","ts":"2026-09-18T09:45:27.587Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"4e732748-e617-4213-b19a-1f4381b29f42","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:27.591Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"7e36f4ad-1d6a-4e7e-9bc8-8df4888e0f1b","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.591Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"3c212a9b-aef6-4b0b-8130-8a0c64ebe242","namespacedName":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:27.591Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"3c212a9b-aef6-4b0b-8130-8a0c64ebe242","mode":"IntegratedOAuth","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.591Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"3c212a9b-aef6-4b0b-8130-8a0c64ebe242","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.666Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b97fc925-63dc-4555-8c4d-71abda707077","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:27.666Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b97fc925-63dc-4555-8c4d-71abda707077","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.666Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b97fc925-63dc-4555-8c4d-71abda707077","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:27.666Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b97fc925-63dc-4555-8c4d-71abda707077"}
{"level":"info","ts":"2026-09-18T09:45:27.666Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b97fc925-63dc-4555-8c4d-71abda707077","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:27.666Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b97fc925-63dc-4555-8c4d-71abda707077","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:27.666Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b97fc925-63dc-4555-8c4d-71abda707077","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:45:27.666Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"fd83db1d-a1ff-4337-9757-c98e8fbf23d0","rayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.666Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"fd83db1d-a1ff-4337-9757-c98e8fbf23d0"}
{"level":"info","ts":"2026-09-18T09:45:27.667Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"224216da-cb40-476c-8e7a-d2c914135a35","name":"counter-wmhkk-workers"}
{"level":"info","ts":"2026-09-18T09:45:27.667Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"224216da-cb40-476c-8e7a-d2c914135a35","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.667Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"3627f2ca-1429-4592-981a-63846651d143","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.667Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"142c078f-b421-4916-85d2-97cce21ee896"}
{"level":"info","ts":"2026-09-18T09:45:27.667Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"142c078f-b421-4916-85d2-97cce21ee896"}
{"level":"info","ts":"2026-09-18T09:45:27.667Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"142c078f-b421-4916-85d2-97cce21ee896"}
{"level":"info","ts":"2026-09-18T09:45:27.667Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"142c078f-b421-4916-85d2-97cce21ee896","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:27.667Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"142c078f-b421-4916-85d2-97cce21ee896","error":"head certificate secret ray-head-secret-counter-wmhkk not found"}
{"level":"error","ts":"2026-09-18T09:45:27.667Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"142c078f-b421-4916-85d2-97cce21ee896","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:27.667Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"142c078f-b421-4916-85d2-97cce21ee896","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found"}
{"level":"info","ts":"2026-09-18T09:45:27.668Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"142c078f-b421-4916-85d2-97cce21ee896","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.668Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"142c078f-b421-4916-85d2-97cce21ee896","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.668Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"142c078f-b421-4916-85d2-97cce21ee896","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.668Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"142c078f-b421-4916-85d2-97cce21ee896"}
{"level":"error","ts":"2026-09-18T09:45:27.668Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"142c078f-b421-4916-85d2-97cce21ee896","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:27.768Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"59488bba-61e4-48aa-b0af-93598ba92747","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:27.768Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"59488bba-61e4-48aa-b0af-93598ba92747","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.768Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"59488bba-61e4-48aa-b0af-93598ba92747","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:27.768Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"59488bba-61e4-48aa-b0af-93598ba92747"}
{"level":"info","ts":"2026-09-18T09:45:27.768Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"59488bba-61e4-48aa-b0af-93598ba92747","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:27.768Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"59488bba-61e4-48aa-b0af-93598ba92747","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:27.768Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"3c212a9b-aef6-4b0b-8130-8a0c64ebe242","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:27.866Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"284d7741-f21a-492c-bbeb-63184466cab0"}
{"level":"info","ts":"2026-09-18T09:45:27.866Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"284d7741-f21a-492c-bbeb-63184466cab0"}
{"level":"info","ts":"2026-09-18T09:45:27.866Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"284d7741-f21a-492c-bbeb-63184466cab0"}
{"level":"info","ts":"2026-09-18T09:45:27.866Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"284d7741-f21a-492c-bbeb-63184466cab0","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:27.867Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"284d7741-f21a-492c-bbeb-63184466cab0","error":"head certificate secret ray-head-secret-counter-wmhkk not found"}
{"level":"error","ts":"2026-09-18T09:45:27.867Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"284d7741-f21a-492c-bbeb-63184466cab0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:27.867Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"284d7741-f21a-492c-bbeb-63184466cab0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found"}
{"level":"info","ts":"2026-09-18T09:45:27.867Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"284d7741-f21a-492c-bbeb-63184466cab0","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.867Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"284d7741-f21a-492c-bbeb-63184466cab0","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.867Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"284d7741-f21a-492c-bbeb-63184466cab0","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:27.867Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"284d7741-f21a-492c-bbeb-63184466cab0"}
{"level":"error","ts":"2026-09-18T09:45:27.867Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"284d7741-f21a-492c-bbeb-63184466cab0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:27.873Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"3627f2ca-1429-4592-981a-63846651d143","name":"counter-wmhkk-head"}
{"level":"info","ts":"2026-09-18T09:45:27.970Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"3627f2ca-1429-4592-981a-63846651d143","name":"counter-wmhkk-workers"}
{"level":"info","ts":"2026-09-18T09:45:27.970Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"3627f2ca-1429-4592-981a-63846651d143","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:28.188Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c57d18e9-b799-4a0c-a8a3-b10812d5282e"}
{"level":"info","ts":"2026-09-18T09:45:28.188Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c57d18e9-b799-4a0c-a8a3-b10812d5282e"}
{"level":"info","ts":"2026-09-18T09:45:28.188Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c57d18e9-b799-4a0c-a8a3-b10812d5282e"}
{"level":"info","ts":"2026-09-18T09:45:28.188Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c57d18e9-b799-4a0c-a8a3-b10812d5282e","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:28.188Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c57d18e9-b799-4a0c-a8a3-b10812d5282e","error":"head certificate secret ray-head-secret-counter-wmhkk not found"}
{"level":"error","ts":"2026-09-18T09:45:28.188Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c57d18e9-b799-4a0c-a8a3-b10812d5282e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:28.188Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c57d18e9-b799-4a0c-a8a3-b10812d5282e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found"}
{"level":"info","ts":"2026-09-18T09:45:28.188Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c57d18e9-b799-4a0c-a8a3-b10812d5282e","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:28.188Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c57d18e9-b799-4a0c-a8a3-b10812d5282e","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:28.188Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c57d18e9-b799-4a0c-a8a3-b10812d5282e","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:28.188Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c57d18e9-b799-4a0c-a8a3-b10812d5282e"}
{"level":"error","ts":"2026-09-18T09:45:28.188Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c57d18e9-b799-4a0c-a8a3-b10812d5282e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-wmhkk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:28.829Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3414905a-9971-46e7-897f-21866515fd83"}
{"level":"info","ts":"2026-09-18T09:45:28.829Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3414905a-9971-46e7-897f-21866515fd83"}
{"level":"info","ts":"2026-09-18T09:45:28.829Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3414905a-9971-46e7-897f-21866515fd83"}
{"level":"info","ts":"2026-09-18T09:45:28.829Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3414905a-9971-46e7-897f-21866515fd83","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:28.829Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3414905a-9971-46e7-897f-21866515fd83","error":"worker certificate secret ray-worker-secret-counter-wmhkk not found"}
{"level":"error","ts":"2026-09-18T09:45:28.829Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3414905a-9971-46e7-897f-21866515fd83","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-counter-wmhkk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:28.829Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3414905a-9971-46e7-897f-21866515fd83","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-counter-wmhkk not found"}
{"level":"info","ts":"2026-09-18T09:45:28.829Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3414905a-9971-46e7-897f-21866515fd83","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:28.829Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3414905a-9971-46e7-897f-21866515fd83","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:28.829Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3414905a-9971-46e7-897f-21866515fd83","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-wmhkk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:28.829Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3414905a-9971-46e7-897f-21866515fd83"}
{"level":"error","ts":"2026-09-18T09:45:28.829Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"3414905a-9971-46e7-897f-21866515fd83","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-counter-wmhkk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:30.108Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"2899a91f-7bc2-4ee9-98a6-89582c6bc031","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:30.108Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"2899a91f-7bc2-4ee9-98a6-89582c6bc031","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:30.108Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"2899a91f-7bc2-4ee9-98a6-89582c6bc031","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:30.109Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505"}
{"level":"info","ts":"2026-09-18T09:45:30.109Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505"}
{"level":"info","ts":"2026-09-18T09:45:30.109Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505"}
{"level":"info","ts":"2026-09-18T09:45:30.109Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:30.110Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505"}
{"level":"info","ts":"2026-09-18T09:45:30.110Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:45:30.110Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505"}
{"level":"info","ts":"2026-09-18T09:45:30.110Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505","mode":"IntegratedOAuth","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:30.110Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505","cluster":"counter-wmhkk","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:45:30.110Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505","cluster":"counter-wmhkk","authType":"IntegratedOAuth","serviceAccount":"counter-wmhkk-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:45:30.110Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-wmhkk","ray.io/group":"headgroup","ray.io/identifier":"counter-wmhkk-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:45:30.110Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","num-cpus":"8","num-gpus":"4","resources":"'{\"R1\": 4}'"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:45:30.110Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=8  --num-gpus=4  --resources='{\"R1\": 4}' "}
{"level":"info","ts":"2026-09-18T09:45:30.110Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=8  --num-gpus=4  --resources='{\"R1\": 4}' "}
{"level":"info","ts":"2026-09-18T09:45:30.110Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505","enabled":true}
{"level":"info","ts":"2026-09-18T09:45:30.111Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"2899a91f-7bc2-4ee9-98a6-89582c6bc031","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:25Z","endTime":"2026-09-18T09:45:27Z"},"jobId":"fail-mvhhs","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:45:25,700\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mvhhs: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:45:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:25Z","endTime":"2026-09-18T09:45:27Z"},"jobId":"fail-mvhhs","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:45:25,700\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mvhhs: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:45:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:30.129Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505","name":"counter-wmhkk-head-j9258"}
{"level":"info","ts":"2026-09-18T09:45:30.129Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505","podName":"counter-wmhkk-head-j9258","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:45:30.129Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505","podName":"counter-wmhkk-head-j9258","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:45:30.129Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c754061c-92fe-4c5c-8f5b-4439459a5505","name":"counter-wmhkk","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:30.142Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"fc6d5bae-6aeb-454b-b14f-04375187d5fa"}
{"level":"info","ts":"2026-09-18T09:45:30.142Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"fc6d5bae-6aeb-454b-b14f-04375187d5fa"}
{"level":"info","ts":"2026-09-18T09:45:30.142Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"fc6d5bae-6aeb-454b-b14f-04375187d5fa"}
{"level":"info","ts":"2026-09-18T09:45:30.142Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"fc6d5bae-6aeb-454b-b14f-04375187d5fa","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:30.142Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"c91858d4-9c9f-4fd4-8b4a-cdcbb5875d60","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:30.142Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"c91858d4-9c9f-4fd4-8b4a-cdcbb5875d60","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:30.142Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"c91858d4-9c9f-4fd4-8b4a-cdcbb5875d60","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:30.142Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"37f9422c-996e-49ac-8f87-ea60f2e1d63f","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:30.142Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"c91858d4-9c9f-4fd4-8b4a-cdcbb5875d60"}
{"level":"info","ts":"2026-09-18T09:45:30.142Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"c91858d4-9c9f-4fd4-8b4a-cdcbb5875d60","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:30.142Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"c91858d4-9c9f-4fd4-8b4a-cdcbb5875d60","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:30.142Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"c91858d4-9c9f-4fd4-8b4a-cdcbb5875d60","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:45:30.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"fc6d5bae-6aeb-454b-b14f-04375187d5fa","Found 1 head Pod":"counter-wmhkk-head-j9258","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:30.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"fc6d5bae-6aeb-454b-b14f-04375187d5fa","head Pod":"counter-wmhkk-head-j9258","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-wmhkk-head-j9258. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:30.143Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"fc6d5bae-6aeb-454b-b14f-04375187d5fa","podName":"counter-wmhkk-head-j9258","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:45:30.143Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"fc6d5bae-6aeb-454b-b14f-04375187d5fa","podName":"counter-wmhkk-head-j9258","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:45:30.143Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"83ad857b-4a47-4030-a490-2718fe6534fe","rayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:30.143Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"fc6d5bae-6aeb-454b-b14f-04375187d5fa","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:45:30.143Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"fc6d5bae-6aeb-454b-b14f-04375187d5fa","seconds":300}
{"level":"info","ts":"2026-09-18T09:45:30.143Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"83ad857b-4a47-4030-a490-2718fe6534fe","rayCluster":"counter-wmhkk","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:45:30.154Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"37f9422c-996e-49ac-8f87-ea60f2e1d63f","name":"counter-wmhkk-head"}
{"level":"info","ts":"2026-09-18T09:45:30.155Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6e8cad94-ee84-4c4e-96d1-4e507d75d1bc"}
{"level":"info","ts":"2026-09-18T09:45:30.155Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6e8cad94-ee84-4c4e-96d1-4e507d75d1bc"}
{"level":"info","ts":"2026-09-18T09:45:30.155Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6e8cad94-ee84-4c4e-96d1-4e507d75d1bc"}
{"level":"info","ts":"2026-09-18T09:45:30.155Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6e8cad94-ee84-4c4e-96d1-4e507d75d1bc","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:30.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6e8cad94-ee84-4c4e-96d1-4e507d75d1bc","Found 1 head Pod":"counter-wmhkk-head-j9258","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:30.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6e8cad94-ee84-4c4e-96d1-4e507d75d1bc","head Pod":"counter-wmhkk-head-j9258","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-wmhkk-head-j9258. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:30.155Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6e8cad94-ee84-4c4e-96d1-4e507d75d1bc","podName":"counter-wmhkk-head-j9258","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:45:30.155Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6e8cad94-ee84-4c4e-96d1-4e507d75d1bc","podName":"counter-wmhkk-head-j9258","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:45:30.155Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6e8cad94-ee84-4c4e-96d1-4e507d75d1bc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:45:30.155Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6e8cad94-ee84-4c4e-96d1-4e507d75d1bc","seconds":300}
{"level":"info","ts":"2026-09-18T09:45:30.166Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"589961c1-aa01-4112-b8bc-dd23ad8fbc7d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:30.166Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"589961c1-aa01-4112-b8bc-dd23ad8fbc7d","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:30.166Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"589961c1-aa01-4112-b8bc-dd23ad8fbc7d","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:30.166Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"589961c1-aa01-4112-b8bc-dd23ad8fbc7d"}
{"level":"info","ts":"2026-09-18T09:45:30.166Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"589961c1-aa01-4112-b8bc-dd23ad8fbc7d","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:30.166Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"589961c1-aa01-4112-b8bc-dd23ad8fbc7d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:30.175Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"37f9422c-996e-49ac-8f87-ea60f2e1d63f","name":"counter-wmhkk-workers"}
{"level":"info","ts":"2026-09-18T09:45:30.175Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"37f9422c-996e-49ac-8f87-ea60f2e1d63f","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:30.188Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"058f2a58-f2a9-459b-87df-cdd7ab6b4e36"}
{"level":"info","ts":"2026-09-18T09:45:30.189Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"058f2a58-f2a9-459b-87df-cdd7ab6b4e36"}
{"level":"info","ts":"2026-09-18T09:45:30.189Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"058f2a58-f2a9-459b-87df-cdd7ab6b4e36"}
{"level":"info","ts":"2026-09-18T09:45:30.189Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"058f2a58-f2a9-459b-87df-cdd7ab6b4e36","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:30.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"058f2a58-f2a9-459b-87df-cdd7ab6b4e36","Found 1 head Pod":"counter-wmhkk-head-j9258","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:30.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"058f2a58-f2a9-459b-87df-cdd7ab6b4e36","head Pod":"counter-wmhkk-head-j9258","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-wmhkk-head-j9258. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:30.189Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"058f2a58-f2a9-459b-87df-cdd7ab6b4e36","podName":"counter-wmhkk-head-j9258","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:45:30.189Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"058f2a58-f2a9-459b-87df-cdd7ab6b4e36","podName":"counter-wmhkk-head-j9258","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:45:30.189Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"058f2a58-f2a9-459b-87df-cdd7ab6b4e36","name":"counter-wmhkk","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:30.204Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"92ddab74-7cfa-41f7-b2e1-0a89b4534c1c","rayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:30.204Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"853af7d1-e5b4-49fc-a7d8-d5c0ee29844b","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:30.204Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"92ddab74-7cfa-41f7-b2e1-0a89b4534c1c","rayCluster":"counter-wmhkk","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:45:30.205Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"4941cffc-116f-41f5-a24b-806101f1ac14","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:30.205Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"4941cffc-116f-41f5-a24b-806101f1ac14","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:30.205Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"4941cffc-116f-41f5-a24b-806101f1ac14","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:30.205Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"4941cffc-116f-41f5-a24b-806101f1ac14"}
{"level":"info","ts":"2026-09-18T09:45:30.205Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"4941cffc-116f-41f5-a24b-806101f1ac14","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:30.205Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"4941cffc-116f-41f5-a24b-806101f1ac14","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:30.205Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"4941cffc-116f-41f5-a24b-806101f1ac14","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:45:30.225Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b003d2b0-6f03-432a-9c45-0814cdfde4e3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:30.225Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b003d2b0-6f03-432a-9c45-0814cdfde4e3","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:30.225Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b003d2b0-6f03-432a-9c45-0814cdfde4e3","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:30.225Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b003d2b0-6f03-432a-9c45-0814cdfde4e3"}
{"level":"info","ts":"2026-09-18T09:45:30.225Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b003d2b0-6f03-432a-9c45-0814cdfde4e3","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:30.225Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b003d2b0-6f03-432a-9c45-0814cdfde4e3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:30.241Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"853af7d1-e5b4-49fc-a7d8-d5c0ee29844b","name":"counter-wmhkk-head"}
{"level":"info","ts":"2026-09-18T09:45:30.254Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"853af7d1-e5b4-49fc-a7d8-d5c0ee29844b","name":"counter-wmhkk-workers"}
{"level":"info","ts":"2026-09-18T09:45:30.254Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"853af7d1-e5b4-49fc-a7d8-d5c0ee29844b","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:30.378Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"02300954-a838-4cf4-88a0-c189887f6b08","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:30.378Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"02300954-a838-4cf4-88a0-c189887f6b08","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:30.378Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"02300954-a838-4cf4-88a0-c189887f6b08","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:30.378Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"02300954-a838-4cf4-88a0-c189887f6b08"}
{"level":"info","ts":"2026-09-18T09:45:30.378Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"02300954-a838-4cf4-88a0-c189887f6b08","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:30.378Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"02300954-a838-4cf4-88a0-c189887f6b08","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:30.738Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"e64bb88f-c6f4-4a1d-9498-373c73c56b20"}
{"level":"info","ts":"2026-09-18T09:45:30.738Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"e64bb88f-c6f4-4a1d-9498-373c73c56b20"}
{"level":"info","ts":"2026-09-18T09:45:30.738Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"e64bb88f-c6f4-4a1d-9498-373c73c56b20"}
{"level":"info","ts":"2026-09-18T09:45:30.738Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"e64bb88f-c6f4-4a1d-9498-373c73c56b20","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:30.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"e64bb88f-c6f4-4a1d-9498-373c73c56b20","Found 1 head Pod":"counter-wmhkk-head-j9258","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:30.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"e64bb88f-c6f4-4a1d-9498-373c73c56b20","head Pod":"counter-wmhkk-head-j9258","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-wmhkk-head-j9258. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:30.738Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"e64bb88f-c6f4-4a1d-9498-373c73c56b20","podName":"counter-wmhkk-head-j9258","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:45:30.738Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"e64bb88f-c6f4-4a1d-9498-373c73c56b20","podName":"counter-wmhkk-head-j9258","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:45:30.739Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"e64bb88f-c6f4-4a1d-9498-373c73c56b20","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:45:30.739Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"e64bb88f-c6f4-4a1d-9498-373c73c56b20","seconds":300}
{"level":"info","ts":"2026-09-18T09:45:30.960Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7802d83-7f09-4db2-8641-7ada7bbb0ac4"}
{"level":"info","ts":"2026-09-18T09:45:30.960Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7802d83-7f09-4db2-8641-7ada7bbb0ac4"}
{"level":"info","ts":"2026-09-18T09:45:30.960Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7802d83-7f09-4db2-8641-7ada7bbb0ac4"}
{"level":"info","ts":"2026-09-18T09:45:30.960Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7802d83-7f09-4db2-8641-7ada7bbb0ac4","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:30.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7802d83-7f09-4db2-8641-7ada7bbb0ac4","Found 1 head Pod":"counter-wmhkk-head-j9258","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:30.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7802d83-7f09-4db2-8641-7ada7bbb0ac4","head Pod":"counter-wmhkk-head-j9258","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-wmhkk-head-j9258. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:30.960Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7802d83-7f09-4db2-8641-7ada7bbb0ac4","podName":"counter-wmhkk-head-j9258","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:45:30.960Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7802d83-7f09-4db2-8641-7ada7bbb0ac4","podName":"counter-wmhkk-head-j9258","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:45:30.960Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7802d83-7f09-4db2-8641-7ada7bbb0ac4","name":"counter-wmhkk","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:30.974Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"43d2336d-64bd-45ec-8cb0-fd4b339ca4df","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:30.974Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9df132ef-e059-462e-8d7c-9c016286650d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:30.974Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9df132ef-e059-462e-8d7c-9c016286650d","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:30.974Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9df132ef-e059-462e-8d7c-9c016286650d","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:30.974Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9df132ef-e059-462e-8d7c-9c016286650d"}
{"level":"info","ts":"2026-09-18T09:45:30.974Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9df132ef-e059-462e-8d7c-9c016286650d","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:30.974Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9df132ef-e059-462e-8d7c-9c016286650d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:30.974Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9df132ef-e059-462e-8d7c-9c016286650d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:45:30.975Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"f84005c5-6610-411e-8bd4-5c17d4018016","rayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:30.975Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"f84005c5-6610-411e-8bd4-5c17d4018016","pod":"counter-wmhkk-head-j9258","ip":"10.133.0.28"}
{"level":"info","ts":"2026-09-18T09:45:30.986Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"43d2336d-64bd-45ec-8cb0-fd4b339ca4df","name":"counter-wmhkk-head"}
{"level":"info","ts":"2026-09-18T09:45:30.988Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"a43c21c7-a2c0-4f06-8b34-d99f493d55a3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:30.988Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"a43c21c7-a2c0-4f06-8b34-d99f493d55a3","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:30.988Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"a43c21c7-a2c0-4f06-8b34-d99f493d55a3","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:30.988Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"a43c21c7-a2c0-4f06-8b34-d99f493d55a3"}
{"level":"info","ts":"2026-09-18T09:45:30.988Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"a43c21c7-a2c0-4f06-8b34-d99f493d55a3","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:30.988Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"a43c21c7-a2c0-4f06-8b34-d99f493d55a3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:30.990Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"f84005c5-6610-411e-8bd4-5c17d4018016","certificate":"ray-head-cert-counter-wmhkk","dnsNames":["counter-wmhkk-head-svc","localhost","counter-wmhkk-head-svc.test-ns-q2lj2.svc","counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local"],"ipAddresses":["10.133.0.28","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:45:30.996Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"43d2336d-64bd-45ec-8cb0-fd4b339ca4df","name":"counter-wmhkk-workers"}
{"level":"info","ts":"2026-09-18T09:45:30.996Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"43d2336d-64bd-45ec-8cb0-fd4b339ca4df","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:31.003Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"f84005c5-6610-411e-8bd4-5c17d4018016","certificate":"ray-worker-cert-counter-wmhkk","dnsNames":["counter-wmhkk-worker-svc","localhost","counter-wmhkk-worker-svc.test-ns-q2lj2.svc","counter-wmhkk-worker-svc.test-ns-q2lj2.svc.cluster.local","*.counter-wmhkk-worker-svc.test-ns-q2lj2.svc","*.counter-wmhkk-worker-svc.test-ns-q2lj2.svc.cluster.local","*-worker-*.test-ns-q2lj2.svc","*-worker-*.test-ns-q2lj2.svc.cluster.local"],"ipAddresses":["10.133.0.28","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:45:31.003Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"f84005c5-6610-411e-8bd4-5c17d4018016","rayCluster":"counter-wmhkk","podIPs":["10.133.0.28"]}
{"level":"info","ts":"2026-09-18T09:45:32.142Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7b9f070-fe84-4941-812d-d4a9622ed91d"}
{"level":"info","ts":"2026-09-18T09:45:32.142Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7b9f070-fe84-4941-812d-d4a9622ed91d"}
{"level":"info","ts":"2026-09-18T09:45:32.142Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7b9f070-fe84-4941-812d-d4a9622ed91d"}
{"level":"info","ts":"2026-09-18T09:45:32.142Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7b9f070-fe84-4941-812d-d4a9622ed91d","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:32.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7b9f070-fe84-4941-812d-d4a9622ed91d","Found 1 head Pod":"counter-wmhkk-head-j9258","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:32.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7b9f070-fe84-4941-812d-d4a9622ed91d","head Pod":"counter-wmhkk-head-j9258","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-wmhkk-head-j9258. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:32.143Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7b9f070-fe84-4941-812d-d4a9622ed91d","podName":"counter-wmhkk-head-j9258","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:45:32.143Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7b9f070-fe84-4941-812d-d4a9622ed91d","podName":"counter-wmhkk-head-j9258","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:45:32.143Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7b9f070-fe84-4941-812d-d4a9622ed91d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:45:32.143Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"d7b9f070-fe84-4941-812d-d4a9622ed91d","seconds":300}
{"level":"info","ts":"2026-09-18T09:45:32.343Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"87a262ff-6cf0-4de0-8623-629d3477d108"}
{"level":"info","ts":"2026-09-18T09:45:32.343Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"87a262ff-6cf0-4de0-8623-629d3477d108"}
{"level":"info","ts":"2026-09-18T09:45:32.343Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"87a262ff-6cf0-4de0-8623-629d3477d108"}
{"level":"info","ts":"2026-09-18T09:45:32.343Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"87a262ff-6cf0-4de0-8623-629d3477d108","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:32.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"87a262ff-6cf0-4de0-8623-629d3477d108","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:32.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"87a262ff-6cf0-4de0-8623-629d3477d108","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:32.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"87a262ff-6cf0-4de0-8623-629d3477d108","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:45:32.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"87a262ff-6cf0-4de0-8623-629d3477d108","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:32.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"87a262ff-6cf0-4de0-8623-629d3477d108","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:32.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"87a262ff-6cf0-4de0-8623-629d3477d108","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:45:32.343Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"87a262ff-6cf0-4de0-8623-629d3477d108","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:45:32.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"87a262ff-6cf0-4de0-8623-629d3477d108","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:32.343Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"87a262ff-6cf0-4de0-8623-629d3477d108","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:45:32.343Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"87a262ff-6cf0-4de0-8623-629d3477d108","seconds":300}
{"level":"info","ts":"2026-09-18T09:45:32.360Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b3a22164-9d86-480e-8217-3ce9a8a149de"}
{"level":"info","ts":"2026-09-18T09:45:32.360Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b3a22164-9d86-480e-8217-3ce9a8a149de"}
{"level":"info","ts":"2026-09-18T09:45:32.360Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b3a22164-9d86-480e-8217-3ce9a8a149de"}
{"level":"info","ts":"2026-09-18T09:45:32.360Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b3a22164-9d86-480e-8217-3ce9a8a149de","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:32.360Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b3a22164-9d86-480e-8217-3ce9a8a149de","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:32.360Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b3a22164-9d86-480e-8217-3ce9a8a149de","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:32.360Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b3a22164-9d86-480e-8217-3ce9a8a149de","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:45:32.360Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b3a22164-9d86-480e-8217-3ce9a8a149de","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:32.360Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b3a22164-9d86-480e-8217-3ce9a8a149de","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:32.360Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b3a22164-9d86-480e-8217-3ce9a8a149de","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:45:32.360Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b3a22164-9d86-480e-8217-3ce9a8a149de","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:45:32.360Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b3a22164-9d86-480e-8217-3ce9a8a149de","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:32.360Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b3a22164-9d86-480e-8217-3ce9a8a149de","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:45:32.360Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b3a22164-9d86-480e-8217-3ce9a8a149de","seconds":300}
{"level":"info","ts":"2026-09-18T09:45:32.378Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13ecddd6-5d12-4ad5-8595-c3e9ce2cea42"}
{"level":"info","ts":"2026-09-18T09:45:32.378Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13ecddd6-5d12-4ad5-8595-c3e9ce2cea42"}
{"level":"info","ts":"2026-09-18T09:45:32.378Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13ecddd6-5d12-4ad5-8595-c3e9ce2cea42"}
{"level":"info","ts":"2026-09-18T09:45:32.378Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13ecddd6-5d12-4ad5-8595-c3e9ce2cea42","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:32.378Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13ecddd6-5d12-4ad5-8595-c3e9ce2cea42","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:32.378Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13ecddd6-5d12-4ad5-8595-c3e9ce2cea42","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:32.378Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13ecddd6-5d12-4ad5-8595-c3e9ce2cea42","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:45:32.378Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13ecddd6-5d12-4ad5-8595-c3e9ce2cea42","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:32.378Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13ecddd6-5d12-4ad5-8595-c3e9ce2cea42","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:32.378Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13ecddd6-5d12-4ad5-8595-c3e9ce2cea42","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:45:32.378Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13ecddd6-5d12-4ad5-8595-c3e9ce2cea42","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:45:32.378Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13ecddd6-5d12-4ad5-8595-c3e9ce2cea42","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:32.379Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13ecddd6-5d12-4ad5-8595-c3e9ce2cea42","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:45:32.379Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13ecddd6-5d12-4ad5-8595-c3e9ce2cea42","seconds":300}
{"level":"info","ts":"2026-09-18T09:45:32.398Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"74f9087e-1a0b-4b07-8888-397daf8d4006"}
{"level":"info","ts":"2026-09-18T09:45:32.398Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"74f9087e-1a0b-4b07-8888-397daf8d4006"}
{"level":"info","ts":"2026-09-18T09:45:32.398Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"74f9087e-1a0b-4b07-8888-397daf8d4006"}
{"level":"info","ts":"2026-09-18T09:45:32.398Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"74f9087e-1a0b-4b07-8888-397daf8d4006","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:32.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"74f9087e-1a0b-4b07-8888-397daf8d4006","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:32.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"74f9087e-1a0b-4b07-8888-397daf8d4006","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:32.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"74f9087e-1a0b-4b07-8888-397daf8d4006","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:45:32.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"74f9087e-1a0b-4b07-8888-397daf8d4006","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-cjnn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:32.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"74f9087e-1a0b-4b07-8888-397daf8d4006","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:32.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"74f9087e-1a0b-4b07-8888-397daf8d4006","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:45:32.398Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"74f9087e-1a0b-4b07-8888-397daf8d4006","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:45:32.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"74f9087e-1a0b-4b07-8888-397daf8d4006","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:32.398Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"74f9087e-1a0b-4b07-8888-397daf8d4006","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:45:32.398Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"74f9087e-1a0b-4b07-8888-397daf8d4006","seconds":300}
{"level":"info","ts":"2026-09-18T09:45:32.940Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0518b9f-5d85-4d84-a75f-ec20f773fdb7"}
{"level":"info","ts":"2026-09-18T09:45:32.940Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0518b9f-5d85-4d84-a75f-ec20f773fdb7"}
{"level":"info","ts":"2026-09-18T09:45:32.940Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0518b9f-5d85-4d84-a75f-ec20f773fdb7"}
{"level":"info","ts":"2026-09-18T09:45:32.940Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0518b9f-5d85-4d84-a75f-ec20f773fdb7","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:32.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0518b9f-5d85-4d84-a75f-ec20f773fdb7","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:32.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0518b9f-5d85-4d84-a75f-ec20f773fdb7","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:32.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0518b9f-5d85-4d84-a75f-ec20f773fdb7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:45:32.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0518b9f-5d85-4d84-a75f-ec20f773fdb7","worker Pod":"raycluster-small-group-worker-cjnn9","shouldDelete":true,"reason":"The worker Pod raycluster-small-group-worker-cjnn9 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-09-18T09:45:32.945Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0518b9f-5d85-4d84-a75f-ec20f773fdb7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:32.945Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0518b9f-5d85-4d84-a75f-ec20f773fdb7","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-09-18T09:45:32.945Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0518b9f-5d85-4d84-a75f-ec20f773fdb7","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:44:59Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:32Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:32.962Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0518b9f-5d85-4d84-a75f-ec20f773fdb7"}
{"level":"error","ts":"2026-09-18T09:45:32.962Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"b0518b9f-5d85-4d84-a75f-ec20f773fdb7","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:32.962Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"10cd2fb9-a89f-4d47-9d7f-e02ca221b60c","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:32.962Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"6e9a4555-2abf-4ecf-a3d4-8bf379fa4dce","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:32.962Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"10cd2fb9-a89f-4d47-9d7f-e02ca221b60c","pod":"raycluster-head-2vjtt","ip":"10.132.0.30"}
{"level":"info","ts":"2026-09-18T09:45:32.962Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"10cd2fb9-a89f-4d47-9d7f-e02ca221b60c","pod":"raycluster-small-group-worker-cjnn9","ip":"10.133.0.25"}
{"level":"info","ts":"2026-09-18T09:45:32.962Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"10cd2fb9-a89f-4d47-9d7f-e02ca221b60c","rayCluster":"raycluster","podIPs":["10.132.0.30","10.133.0.25"]}
{"level":"info","ts":"2026-09-18T09:45:32.968Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"6ca27c4a-ca92-444d-9fc5-1da93d1433eb"}
{"level":"info","ts":"2026-09-18T09:45:32.968Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"6ca27c4a-ca92-444d-9fc5-1da93d1433eb"}
{"level":"info","ts":"2026-09-18T09:45:32.968Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"6ca27c4a-ca92-444d-9fc5-1da93d1433eb"}
{"level":"info","ts":"2026-09-18T09:45:32.968Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"6ca27c4a-ca92-444d-9fc5-1da93d1433eb","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:32.968Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"6ca27c4a-ca92-444d-9fc5-1da93d1433eb","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:32.968Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"6ca27c4a-ca92-444d-9fc5-1da93d1433eb","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:32.968Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"6ca27c4a-ca92-444d-9fc5-1da93d1433eb","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T09:45:32.968Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"6ca27c4a-ca92-444d-9fc5-1da93d1433eb","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-09-18T09:45:32.968Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"6ca27c4a-ca92-444d-9fc5-1da93d1433eb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:45:32.968Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"6ca27c4a-ca92-444d-9fc5-1da93d1433eb","seconds":300}
{"level":"info","ts":"2026-09-18T09:45:32.968Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"6e9a4555-2abf-4ecf-a3d4-8bf379fa4dce","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:33.112Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"68d7182e-f69a-4092-b252-d071311629af","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:33.112Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"68d7182e-f69a-4092-b252-d071311629af","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:33.113Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"68d7182e-f69a-4092-b252-d071311629af","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:33.267Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9454344-ee4b-41ed-a12d-9b92741cc344"}
{"level":"info","ts":"2026-09-18T09:45:33.267Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9454344-ee4b-41ed-a12d-9b92741cc344"}
{"level":"info","ts":"2026-09-18T09:45:33.267Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9454344-ee4b-41ed-a12d-9b92741cc344"}
{"level":"info","ts":"2026-09-18T09:45:33.267Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9454344-ee4b-41ed-a12d-9b92741cc344","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:33.267Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9454344-ee4b-41ed-a12d-9b92741cc344","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:33.267Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9454344-ee4b-41ed-a12d-9b92741cc344","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:33.268Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9454344-ee4b-41ed-a12d-9b92741cc344","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T09:45:33.268Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9454344-ee4b-41ed-a12d-9b92741cc344","podName":"raycluster-small-group-worker-cjnn9","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-09-18T09:45:33.268Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9454344-ee4b-41ed-a12d-9b92741cc344","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:45:33.268Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c9454344-ee4b-41ed-a12d-9b92741cc344","seconds":300}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f"}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f"}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f"}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f"}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:33.272Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","enabled":true}
{"level":"error","ts":"2026-09-18T09:45:33.277Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.277Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:45:33.277Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:32Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:33Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T09:45:33Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:33.290Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f"}
{"level":"error","ts":"2026-09-18T09:45:33.290Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"13cfcec7-41c3-41bd-9528-4dd8b00ec61f","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.291Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"c8a2b429-f102-44c2-9604-ea871b78fb10","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:33.291Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"78cf98a9-f2f0-4167-b28a-781c33387adf","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:33.291Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"c8a2b429-f102-44c2-9604-ea871b78fb10","pod":"raycluster-head-2vjtt","ip":"10.132.0.30"}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593"}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593"}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593"}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593"}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593","enabled":true}
{"level":"info","ts":"2026-09-18T09:45:33.296Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"78cf98a9-f2f0-4167-b28a-781c33387adf","cluster":"raycluster"}
{"level":"error","ts":"2026-09-18T09:45:33.299Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.300Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:45:33.300Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593"}
{"level":"error","ts":"2026-09-18T09:45:33.300Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"0869487d-1b86-4b46-a0fb-69282a5cb593","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.302Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"c8a2b429-f102-44c2-9604-ea871b78fb10","certificate":"ray-head-cert-raycluster","dnsNames":["raycluster-head-svc","localhost","raycluster-head-svc.test-ns-r8v9f.svc","raycluster-head-svc.test-ns-r8v9f.svc.cluster.local"],"ipAddresses":["10.132.0.30","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:45:33.310Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a"}
{"level":"info","ts":"2026-09-18T09:45:33.310Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a"}
{"level":"info","ts":"2026-09-18T09:45:33.310Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a"}
{"level":"info","ts":"2026-09-18T09:45:33.310Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:33.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:33.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:33.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:45:33.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:33.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:45:33.311Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:45:33.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:33.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:45:33.311Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a"}
{"level":"info","ts":"2026-09-18T09:45:33.311Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:45:33.311Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:33.311Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:33.311Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a","enabled":true}
{"level":"error","ts":"2026-09-18T09:45:33.314Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.314Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:45:33.315Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a"}
{"level":"error","ts":"2026-09-18T09:45:33.315Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"50581e0e-6085-4760-8cc1-af8682810a3a","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.316Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"c8a2b429-f102-44c2-9604-ea871b78fb10","certificate":"ray-worker-cert-raycluster","dnsNames":["raycluster-worker-svc","localhost","raycluster-small-group","raycluster-worker-svc.test-ns-r8v9f.svc","raycluster-worker-svc.test-ns-r8v9f.svc.cluster.local","*.raycluster-worker-svc.test-ns-r8v9f.svc","*.raycluster-worker-svc.test-ns-r8v9f.svc.cluster.local","*-worker-*.test-ns-r8v9f.svc","*-worker-*.test-ns-r8v9f.svc.cluster.local"],"ipAddresses":["10.132.0.30","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:45:33.316Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"c8a2b429-f102-44c2-9604-ea871b78fb10","rayCluster":"raycluster","podIPs":["10.132.0.30"]}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978"}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978"}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978"}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978"}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:33.336Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978","enabled":true}
{"level":"error","ts":"2026-09-18T09:45:33.339Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.339Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:45:33.366Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978"}
{"level":"error","ts":"2026-09-18T09:45:33.366Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"bea1d5a0-d781-47fc-9ca4-97a84a072978","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:45:33.406Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"68d7182e-f69a-4092-b252-d071311629af","JobId":"fail-mvhhs","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.406Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"68d7182e-f69a-4092-b252-d071311629af"}
{"level":"info","ts":"2026-09-18T09:45:33.406Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134"}
{"level":"info","ts":"2026-09-18T09:45:33.406Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134"}
{"level":"error","ts":"2026-09-18T09:45:33.406Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"68d7182e-f69a-4092-b252-d071311629af","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.406Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134"}
{"level":"info","ts":"2026-09-18T09:45:33.406Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:33.406Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"d2b50906-e18b-41b5-a4dd-76fb40541945","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:33.406Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"d2b50906-e18b-41b5-a4dd-76fb40541945","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"d2b50906-e18b-41b5-a4dd-76fb40541945","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"d2b50906-e18b-41b5-a4dd-76fb40541945"}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"d2b50906-e18b-41b5-a4dd-76fb40541945","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"d2b50906-e18b-41b5-a4dd-76fb40541945","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134"}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"77109c1a-5896-4dcd-a976-e6fffa1253a3","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134","enabled":true}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"77109c1a-5896-4dcd-a976-e6fffa1253a3","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:33.407Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"77109c1a-5896-4dcd-a976-e6fffa1253a3","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:33.409Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"77109c1a-5896-4dcd-a976-e6fffa1253a3","JobId":"fail-mvhhs","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.409Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"77109c1a-5896-4dcd-a976-e6fffa1253a3"}
{"level":"error","ts":"2026-09-18T09:45:33.409Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"77109c1a-5896-4dcd-a976-e6fffa1253a3","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:45:33.411Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.411Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:45:33.411Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134"}
{"level":"error","ts":"2026-09-18T09:45:33.411Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"9a943d44-9003-4c0b-b5e4-5ef23b796134","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.412Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"801a8675-ff5f-41c2-a174-2f81f750617e","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:33.412Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"801a8675-ff5f-41c2-a174-2f81f750617e","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:33.412Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"801a8675-ff5f-41c2-a174-2f81f750617e","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:33.414Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"801a8675-ff5f-41c2-a174-2f81f750617e","JobId":"fail-mvhhs","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.414Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"801a8675-ff5f-41c2-a174-2f81f750617e"}
{"level":"error","ts":"2026-09-18T09:45:33.414Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"801a8675-ff5f-41c2-a174-2f81f750617e","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.435Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"b78c50d3-0613-40ec-9466-cf330e928154","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:33.435Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"b78c50d3-0613-40ec-9466-cf330e928154","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:33.435Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"b78c50d3-0613-40ec-9466-cf330e928154","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:33.467Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"b78c50d3-0613-40ec-9466-cf330e928154","JobId":"fail-mvhhs","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.467Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"b78c50d3-0613-40ec-9466-cf330e928154"}
{"level":"error","ts":"2026-09-18T09:45:33.467Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"b78c50d3-0613-40ec-9466-cf330e928154","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.491Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0"}
{"level":"info","ts":"2026-09-18T09:45:33.491Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0"}
{"level":"info","ts":"2026-09-18T09:45:33.491Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0"}
{"level":"info","ts":"2026-09-18T09:45:33.491Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:33.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:33.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:33.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:45:33.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:33.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:45:33.492Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:45:33.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:33.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:45:33.492Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0"}
{"level":"info","ts":"2026-09-18T09:45:33.492Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:45:33.492Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:33.492Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:33.492Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0","enabled":true}
{"level":"error","ts":"2026-09-18T09:45:33.496Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.496Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:45:33.496Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0"}
{"level":"error","ts":"2026-09-18T09:45:33.496Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"faff51bf-c1d8-46e1-a448-27fffacf17a0","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.508Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"075ca7dc-423c-4d30-88f2-2871864d1a19","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:33.508Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"075ca7dc-423c-4d30-88f2-2871864d1a19","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:33.508Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"075ca7dc-423c-4d30-88f2-2871864d1a19","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:33.510Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"075ca7dc-423c-4d30-88f2-2871864d1a19","JobId":"fail-mvhhs","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.510Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"075ca7dc-423c-4d30-88f2-2871864d1a19"}
{"level":"error","ts":"2026-09-18T09:45:33.510Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"075ca7dc-423c-4d30-88f2-2871864d1a19","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.590Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"97b61459-cebb-4966-92a6-90617fcb0e32","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:33.590Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"97b61459-cebb-4966-92a6-90617fcb0e32","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:33.590Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"97b61459-cebb-4966-92a6-90617fcb0e32","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:33.593Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"97b61459-cebb-4966-92a6-90617fcb0e32","JobId":"fail-mvhhs","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.593Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"97b61459-cebb-4966-92a6-90617fcb0e32"}
{"level":"error","ts":"2026-09-18T09:45:33.593Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"97b61459-cebb-4966-92a6-90617fcb0e32","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7"}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7"}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7"}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7"}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:33.657Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7","enabled":true}
{"level":"error","ts":"2026-09-18T09:45:33.661Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.661Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:45:33.661Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7"}
{"level":"error","ts":"2026-09-18T09:45:33.661Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f1adaf97-dab1-4ff4-9f89-7131cb7713c7","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.754Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"1a6f0f99-179f-442e-9e50-4b51472900e6","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:33.754Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"1a6f0f99-179f-442e-9e50-4b51472900e6","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:33.754Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"1a6f0f99-179f-442e-9e50-4b51472900e6","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:33.757Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"1a6f0f99-179f-442e-9e50-4b51472900e6","JobId":"fail-mvhhs","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.757Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"1a6f0f99-179f-442e-9e50-4b51472900e6"}
{"level":"error","ts":"2026-09-18T09:45:33.757Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"1a6f0f99-179f-442e-9e50-4b51472900e6","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a"}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a"}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a"}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a"}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:33.982Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a","enabled":true}
{"level":"error","ts":"2026-09-18T09:45:33.987Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:33.987Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:45:33.987Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a"}
{"level":"error","ts":"2026-09-18T09:45:33.987Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"d3604c4e-72bd-4cb1-b65e-bb8f842e805a","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:34.077Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"f09bc51c-10a1-48fe-bbac-1b77776ef2af","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:34.077Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"f09bc51c-10a1-48fe-bbac-1b77776ef2af","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:34.077Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"f09bc51c-10a1-48fe-bbac-1b77776ef2af","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:34.080Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"f09bc51c-10a1-48fe-bbac-1b77776ef2af","JobId":"fail-mvhhs","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:34.080Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"f09bc51c-10a1-48fe-bbac-1b77776ef2af"}
{"level":"error","ts":"2026-09-18T09:45:34.080Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"f09bc51c-10a1-48fe-bbac-1b77776ef2af","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935"}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935"}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935"}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935","head Pod":"raycluster-head-2vjtt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-2vjtt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935"}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:34.628Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935","enabled":true}
{"level":"error","ts":"2026-09-18T09:45:34.632Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:34.632Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:45:34.633Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935"}
{"level":"error","ts":"2026-09-18T09:45:34.633Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"2deee110-3341-4deb-8dd8-dbbb5ae65935","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:34.721Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"4713e124-cce5-408f-9fd5-12b8b7860890","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:34.721Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"4713e124-cce5-408f-9fd5-12b8b7860890","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:34.721Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"4713e124-cce5-408f-9fd5-12b8b7860890","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"error","ts":"2026-09-18T09:45:34.723Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"4713e124-cce5-408f-9fd5-12b8b7860890","JobId":"fail-mvhhs","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:34.723Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"4713e124-cce5-408f-9fd5-12b8b7860890"}
{"level":"error","ts":"2026-09-18T09:45:34.723Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"4713e124-cce5-408f-9fd5-12b8b7860890","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": dial tcp 10.132.0.30:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:34.925Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f837dce8-0337-4e16-8de4-4a7aed814e8b"}
{"level":"info","ts":"2026-09-18T09:45:34.925Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f837dce8-0337-4e16-8de4-4a7aed814e8b"}
{"level":"info","ts":"2026-09-18T09:45:34.925Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f837dce8-0337-4e16-8de4-4a7aed814e8b"}
{"level":"info","ts":"2026-09-18T09:45:34.925Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f837dce8-0337-4e16-8de4-4a7aed814e8b","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:34.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f837dce8-0337-4e16-8de4-4a7aed814e8b","Found 1 head Pod":"raycluster-head-2vjtt","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-09-18 09:43:13 +0000 UTC,FinishedAt:2026-09-18 09:45:34 +0000 UTC,ContainerID:cri-o://2c5e01e3c87bd2584ba8f43d229b5c6ea1f768f76eb2b4cdd3373f141557d32e,}"}
{"level":"info","ts":"2026-09-18T09:45:34.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f837dce8-0337-4e16-8de4-4a7aed814e8b","head Pod":"raycluster-head-2vjtt","shouldDelete":true,"reason":"The head Pod raycluster-head-2vjtt status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-09-18T09:45:34.931Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f837dce8-0337-4e16-8de4-4a7aed814e8b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod raycluster-head-2vjtt status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:34.931Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f837dce8-0337-4e16-8de4-4a7aed814e8b","error":"The head Pod raycluster-head-2vjtt status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-09-18T09:45:34.932Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f837dce8-0337-4e16-8de4-4a7aed814e8b","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:33Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:43:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T09:45:33Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:34Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:34Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T09:45:33Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:34.944Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f837dce8-0337-4e16-8de4-4a7aed814e8b"}
{"level":"error","ts":"2026-09-18T09:45:34.944Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"f837dce8-0337-4e16-8de4-4a7aed814e8b","error":"The head Pod raycluster-head-2vjtt status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:34.944Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"5adf9e6e-1b94-4675-b28c-a326fe1e08bb","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:34.944Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"06bf13ed-6ee9-4530-a0c7-faa089bb443f","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:34.944Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"5adf9e6e-1b94-4675-b28c-a326fe1e08bb","pod":"raycluster-head-2vjtt","ip":"10.132.0.30"}
{"level":"info","ts":"2026-09-18T09:45:34.944Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"5adf9e6e-1b94-4675-b28c-a326fe1e08bb"}
{"level":"info","ts":"2026-09-18T09:45:34.949Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"06bf13ed-6ee9-4530-a0c7-faa089bb443f","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:35.185Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c"}
{"level":"info","ts":"2026-09-18T09:45:35.185Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c"}
{"level":"info","ts":"2026-09-18T09:45:35.185Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c"}
{"level":"info","ts":"2026-09-18T09:45:35.185Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:35.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-09-18T09:45:35.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:45:35.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:35.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:45:35.186Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:45:35.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:45:35.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:45:35.186Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c"}
{"level":"info","ts":"2026-09-18T09:45:35.186Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:45:35.186Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:35.186Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:45:35.186Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","enabled":true}
{"level":"error","ts":"2026-09-18T09:45:35.190Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:35.190Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:45:35.190Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:35.190Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:35.190Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:35.190Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:34Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:34Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T09:45:33Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:35Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:34Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T09:45:33Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c"}
{"level":"error","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"c5298bf9-245d-4daf-89ec-e0b65187116c","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0"}
{"level":"info","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0"}
{"level":"info","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0"}
{"level":"info","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0"}
{"level":"info","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0"}
{"level":"info","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","cluster":"raycluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","cluster":"raycluster","authType":"IntegratedOAuth","serviceAccount":"raycluster-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster","ray.io/group":"headgroup","ray.io/identifier":"raycluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:45:35.266Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","enabled":true}
{"level":"info","ts":"2026-09-18T09:45:35.267Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"0babb2b1-0a97-48e0-85cc-d726e7ac5d50","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:35.268Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"a73c54ee-052e-4fc9-8cd8-28d9473eed30","cluster":"raycluster"}
{"level":"error","ts":"2026-09-18T09:45:35.272Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:35.272Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:45:35.272Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:35.272Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:35.272Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:35.272Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:34Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"raycluster-head-2vjtt","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:34Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T09:45:33Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:35Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:34Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T09:45:33Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:35.274Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"a73c54ee-052e-4fc9-8cd8-28d9473eed30","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:35.281Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"0babb2b1-0a97-48e0-85cc-d726e7ac5d50","certificate":"ray-head-cert-raycluster","dnsNames":["raycluster-head-svc","localhost","raycluster-head-svc.test-ns-r8v9f.svc","raycluster-head-svc.test-ns-r8v9f.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:45:35.281Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-r8v9f","name":"raycluster"}}
{"level":"info","ts":"2026-09-18T09:45:35.281Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0"}
{"level":"error","ts":"2026-09-18T09:45:35.281Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"67b8dabd-d1be-428c-a125-d22466aed6c0","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:35.295Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"0babb2b1-0a97-48e0-85cc-d726e7ac5d50","certificate":"ray-worker-cert-raycluster","dnsNames":["raycluster-worker-svc","localhost","raycluster-small-group","raycluster-worker-svc.test-ns-r8v9f.svc","raycluster-worker-svc.test-ns-r8v9f.svc.cluster.local","*.raycluster-worker-svc.test-ns-r8v9f.svc","*.raycluster-worker-svc.test-ns-r8v9f.svc.cluster.local","*-worker-*.test-ns-r8v9f.svc","*-worker-*.test-ns-r8v9f.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:45:35.295Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"0babb2b1-0a97-48e0-85cc-d726e7ac5d50"}
{"level":"info","ts":"2026-09-18T09:45:35.913Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919"}
{"level":"info","ts":"2026-09-18T09:45:35.913Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919"}
{"level":"info","ts":"2026-09-18T09:45:35.913Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919"}
{"level":"info","ts":"2026-09-18T09:45:35.913Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:35.913Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919"}
{"level":"info","ts":"2026-09-18T09:45:35.913Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:45:35.913Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919"}
{"level":"info","ts":"2026-09-18T09:45:35.913Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:35.913Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","cluster":"raycluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:45:35.913Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","cluster":"raycluster","authType":"IntegratedOAuth","serviceAccount":"raycluster-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:45:35.913Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster","ray.io/group":"headgroup","ray.io/identifier":"raycluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:45:35.913Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:45:35.913Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:45:35.913Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:45:35.913Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","enabled":true}
{"level":"error","ts":"2026-09-18T09:45:35.918Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:35.918Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:45:35.918Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:35.918Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:35.918Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:35.918Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:35Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:34Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T09:45:33Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:35Z","stateTransitionTimes":{"ready":"2026-09-18T09:44:59Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:34Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:44:59Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:40:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:40:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T09:45:33Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:35.932Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919"}
{"level":"error","ts":"2026-09-18T09:45:35.932Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"e0dcfe60-7ba2-4972-914e-c1ebce758919","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:35.932Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"6a0b7dcc-d2ef-4999-ac6f-3bb28cae6408","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:35.932Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"cde720df-543d-4603-9e0c-d23bb5bd3c67","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:35.932Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"6a0b7dcc-d2ef-4999-ac6f-3bb28cae6408"}
{"level":"info","ts":"2026-09-18T09:45:35.938Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"cde720df-543d-4603-9e0c-d23bb5bd3c67","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:35.988Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c0347308-19f1-40cb-adea-98ac928960b8"}
{"level":"info","ts":"2026-09-18T09:45:35.988Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c0347308-19f1-40cb-adea-98ac928960b8"}
{"level":"info","ts":"2026-09-18T09:45:35.988Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c0347308-19f1-40cb-adea-98ac928960b8"}
{"level":"info","ts":"2026-09-18T09:45:35.988Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c0347308-19f1-40cb-adea-98ac928960b8","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:35.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c0347308-19f1-40cb-adea-98ac928960b8","Found 1 head Pod":"counter-wmhkk-head-j9258","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:35.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c0347308-19f1-40cb-adea-98ac928960b8","head Pod":"counter-wmhkk-head-j9258","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-wmhkk-head-j9258. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:35.988Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c0347308-19f1-40cb-adea-98ac928960b8","podName":"counter-wmhkk-head-j9258","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:45:35.988Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c0347308-19f1-40cb-adea-98ac928960b8","podName":"counter-wmhkk-head-j9258","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:45:35.989Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c0347308-19f1-40cb-adea-98ac928960b8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:45:35.989Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"c0347308-19f1-40cb-adea-98ac928960b8","seconds":300}
{"level":"info","ts":"2026-09-18T09:45:36.004Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"f8195707-6efc-41c7-b8ea-7ec66b7c2845","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:45:36.004Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"f8195707-6efc-41c7-b8ea-7ec66b7c2845","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:36.004Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"f8195707-6efc-41c7-b8ea-7ec66b7c2845","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:36.803Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"3b9174b5-4da3-45f2-81ca-cdf96be5d930","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:36.803Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"3b9174b5-4da3-45f2-81ca-cdf96be5d930"}
{"level":"info","ts":"2026-09-18T09:45:37.728Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"2119bf67-d057-49dc-a0d4-17ca7c30df97","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.734Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"2119bf67-d057-49dc-a0d4-17ca7c30df97","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.734Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"0d1167aa-d392-4d2d-95a5-22f6a4feec24","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.739Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"0d1167aa-d392-4d2d-95a5-22f6a4feec24","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.796Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"90f89682-c43c-4d46-8f4c-953478692104","namespacedName":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:37.796Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1031864b-55df-4da9-a240-2bbb707991ce"}
{"level":"info","ts":"2026-09-18T09:45:37.796Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1031864b-55df-4da9-a240-2bbb707991ce"}
{"level":"info","ts":"2026-09-18T09:45:37.796Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"90f89682-c43c-4d46-8f4c-953478692104","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.796Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"90f89682-c43c-4d46-8f4c-953478692104","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.796Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1031864b-55df-4da9-a240-2bbb707991ce"}
{"level":"error","ts":"2026-09-18T09:45:37.799Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1031864b-55df-4da9-a240-2bbb707991ce","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcileHeadService-fm","error":"services \"raycluster-head-svc\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.799Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1031864b-55df-4da9-a240-2bbb707991ce","error":"services \"raycluster-head-svc\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:45:37.800Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1031864b-55df-4da9-a240-2bbb707991ce","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:37.800Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1031864b-55df-4da9-a240-2bbb707991ce","serviceSelectors":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/identifier":"raycluster-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:37.800Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1031864b-55df-4da9-a240-2bbb707991ce","filter labels":["test-ns-r8v9f",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:37.800Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1031864b-55df-4da9-a240-2bbb707991ce","error":"unable to find head service. cluster name raycluster, filter labels [test-ns-r8v9f map[ray.io/cluster:raycluster ray.io/identifier:raycluster-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-09-18T09:45:37.800Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1031864b-55df-4da9-a240-2bbb707991ce"}
{"level":"error","ts":"2026-09-18T09:45:37.800Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"1031864b-55df-4da9-a240-2bbb707991ce","error":"services \"raycluster-head-svc\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:45:37.814Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"90f89682-c43c-4d46-8f4c-953478692104","error":"failed to ensure OIDC resources: failed to ensure ReferenceGrant: failed to create or update ReferenceGrant: referencegrants.gateway.networking.k8s.io \"kuberay-gateway-access\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.814Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"90f89682-c43c-4d46-8f4c-953478692104"}
{"level":"error","ts":"2026-09-18T09:45:37.814Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"90f89682-c43c-4d46-8f4c-953478692104","error":"failed to ensure OIDC resources: failed to ensure ReferenceGrant: failed to create or update ReferenceGrant: referencegrants.gateway.networking.k8s.io \"kuberay-gateway-access\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.814Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"a2e23513-58b6-4e9c-90dd-251a60c8d502","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.814Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"dbee23ed-ddf5-4785-9a57-f27bbc5f9930","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.866Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"dbee23ed-ddf5-4785-9a57-f27bbc5f9930","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.819Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"b1bcdc51-5082-41d9-81af-09a1b5f2d676","namespacedName":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:37.866Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"b1bcdc51-5082-41d9-81af-09a1b5f2d676","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.866Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"b1bcdc51-5082-41d9-81af-09a1b5f2d676","cluster":"raycluster"}
{"level":"error","ts":"2026-09-18T09:45:37.869Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"dbee23ed-ddf5-4785-9a57-f27bbc5f9930","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:45:37.869Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"dbee23ed-ddf5-4785-9a57-f27bbc5f9930","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.869Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"dbee23ed-ddf5-4785-9a57-f27bbc5f9930"}
{"level":"error","ts":"2026-09-18T09:45:37.869Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"dbee23ed-ddf5-4785-9a57-f27bbc5f9930","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.872Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"a2e23513-58b6-4e9c-90dd-251a60c8d502","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.875Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"8735b742-e3c8-4bc7-a5f6-a3985cdd8819","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.875Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"8735b742-e3c8-4bc7-a5f6-a3985cdd8819","rayCluster":"raycluster"}
{"level":"error","ts":"2026-09-18T09:45:37.878Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"b1bcdc51-5082-41d9-81af-09a1b5f2d676","error":"failed to ensure OIDC resources: failed to ensure ReferenceGrant: failed to create or update ReferenceGrant: referencegrants.gateway.networking.k8s.io \"kuberay-gateway-access\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:45:37.878Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"8735b742-e3c8-4bc7-a5f6-a3985cdd8819","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:45:37.878Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"8735b742-e3c8-4bc7-a5f6-a3985cdd8819","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.878Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"8735b742-e3c8-4bc7-a5f6-a3985cdd8819"}
{"level":"error","ts":"2026-09-18T09:45:37.878Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"8735b742-e3c8-4bc7-a5f6-a3985cdd8819","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.878Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"b1bcdc51-5082-41d9-81af-09a1b5f2d676"}
{"level":"error","ts":"2026-09-18T09:45:37.878Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"b1bcdc51-5082-41d9-81af-09a1b5f2d676","error":"failed to ensure OIDC resources: failed to ensure ReferenceGrant: failed to create or update ReferenceGrant: referencegrants.gateway.networking.k8s.io \"kuberay-gateway-access\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.889Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"a5f15cb3-86b6-494a-821d-c4f12e6e511f","namespacedName":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:37.889Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"a5f15cb3-86b6-494a-821d-c4f12e6e511f","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.889Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"a5f15cb3-86b6-494a-821d-c4f12e6e511f","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.889Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"ca0aa24c-e811-4439-8f21-4824111eb3a2","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.889Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"ca0aa24c-e811-4439-8f21-4824111eb3a2","rayCluster":"raycluster"}
{"level":"error","ts":"2026-09-18T09:45:37.892Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"ca0aa24c-e811-4439-8f21-4824111eb3a2","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:45:37.892Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"ca0aa24c-e811-4439-8f21-4824111eb3a2","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.892Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"ca0aa24c-e811-4439-8f21-4824111eb3a2"}
{"level":"error","ts":"2026-09-18T09:45:37.892Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"ca0aa24c-e811-4439-8f21-4824111eb3a2","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:45:37.901Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"a5f15cb3-86b6-494a-821d-c4f12e6e511f","error":"failed to ensure OIDC resources: failed to ensure ReferenceGrant: failed to create or update ReferenceGrant: referencegrants.gateway.networking.k8s.io \"kuberay-gateway-access\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.901Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"a5f15cb3-86b6-494a-821d-c4f12e6e511f"}
{"level":"error","ts":"2026-09-18T09:45:37.901Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"a5f15cb3-86b6-494a-821d-c4f12e6e511f","error":"failed to ensure OIDC resources: failed to ensure ReferenceGrant: failed to create or update ReferenceGrant: referencegrants.gateway.networking.k8s.io \"kuberay-gateway-access\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.912Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"82fc9cbc-4218-4c97-af19-310da9d908bf","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.912Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"82fc9cbc-4218-4c97-af19-310da9d908bf","rayCluster":"raycluster"}
{"level":"error","ts":"2026-09-18T09:45:37.916Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"82fc9cbc-4218-4c97-af19-310da9d908bf","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:45:37.916Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"82fc9cbc-4218-4c97-af19-310da9d908bf","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.916Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"82fc9cbc-4218-4c97-af19-310da9d908bf"}
{"level":"error","ts":"2026-09-18T09:45:37.916Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"82fc9cbc-4218-4c97-af19-310da9d908bf","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.922Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"9df4adf1-ad20-43e1-bd1d-4d534752f743","namespacedName":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:37.922Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"9df4adf1-ad20-43e1-bd1d-4d534752f743","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.922Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"9df4adf1-ad20-43e1-bd1d-4d534752f743","cluster":"raycluster"}
{"level":"error","ts":"2026-09-18T09:45:37.935Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"9df4adf1-ad20-43e1-bd1d-4d534752f743","error":"failed to ensure OIDC resources: failed to ensure ReferenceGrant: failed to create or update ReferenceGrant: referencegrants.gateway.networking.k8s.io \"kuberay-gateway-access\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.935Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"9df4adf1-ad20-43e1-bd1d-4d534752f743"}
{"level":"error","ts":"2026-09-18T09:45:37.935Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"9df4adf1-ad20-43e1-bd1d-4d534752f743","error":"failed to ensure OIDC resources: failed to ensure ReferenceGrant: failed to create or update ReferenceGrant: referencegrants.gateway.networking.k8s.io \"kuberay-gateway-access\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.957Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"2cc0e86e-470c-483f-b2e4-e6d8c5e3ed69","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.957Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"2cc0e86e-470c-483f-b2e4-e6d8c5e3ed69","rayCluster":"raycluster"}
{"level":"error","ts":"2026-09-18T09:45:37.960Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"2cc0e86e-470c-483f-b2e4-e6d8c5e3ed69","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:45:37.960Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"2cc0e86e-470c-483f-b2e4-e6d8c5e3ed69","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.960Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"2cc0e86e-470c-483f-b2e4-e6d8c5e3ed69"}
{"level":"error","ts":"2026-09-18T09:45:37.960Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"2cc0e86e-470c-483f-b2e4-e6d8c5e3ed69","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.976Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d913b42e-7179-442a-9f0c-94688557a400","namespacedName":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:37.976Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d913b42e-7179-442a-9f0c-94688557a400","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:37.976Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d913b42e-7179-442a-9f0c-94688557a400","cluster":"raycluster"}
{"level":"error","ts":"2026-09-18T09:45:37.989Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d913b42e-7179-442a-9f0c-94688557a400","error":"failed to ensure OIDC resources: failed to ensure ReferenceGrant: failed to create or update ReferenceGrant: referencegrants.gateway.networking.k8s.io \"kuberay-gateway-access\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:37.989Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d913b42e-7179-442a-9f0c-94688557a400"}
{"level":"error","ts":"2026-09-18T09:45:37.989Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d913b42e-7179-442a-9f0c-94688557a400","error":"failed to ensure OIDC resources: failed to ensure ReferenceGrant: failed to create or update ReferenceGrant: referencegrants.gateway.networking.k8s.io \"kuberay-gateway-access\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:45:38.005Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"f8195707-6efc-41c7-b8ea-7ec66b7c2845","JobId":"fail-mvhhs","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:38.005Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"f8195707-6efc-41c7-b8ea-7ec66b7c2845"}
{"level":"error","ts":"2026-09-18T09:45:38.005Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"f8195707-6efc-41c7-b8ea-7ec66b7c2845","error":"Get \"http://raycluster-head-svc.test-ns-r8v9f.svc.cluster.local:8265/api/jobs/fail-mvhhs\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:38.005Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"20ee49a6-8e5b-44c1-b293-d350d61af310","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:38.005Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"20ee49a6-8e5b-44c1-b293-d350d61af310","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:38.005Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"20ee49a6-8e5b-44c1-b293-d350d61af310","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:38.005Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"20ee49a6-8e5b-44c1-b293-d350d61af310"}
{"level":"info","ts":"2026-09-18T09:45:38.005Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"20ee49a6-8e5b-44c1-b293-d350d61af310","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:38.005Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"20ee49a6-8e5b-44c1-b293-d350d61af310","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:38.005Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"bb9c5e92-b6dc-4794-87f7-a697c7353696","DeletionTimestamp":"2026-09-18 09:45:37 +0000 UTC"}
{"level":"info","ts":"2026-09-18T09:45:38.006Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"bb9c5e92-b6dc-4794-87f7-a697c7353696","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:45:38.018Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"managed-externally","namespace":"test-ns-r8v9f"},"reconcileID":"083b8aeb-3f16-4105-b7d2-42dfc6c5dd1b"}
{"level":"info","ts":"2026-09-18T09:45:38.018Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"bb805859-11b5-4958-8e63-4107d65baec7","DeletionTimestamp":"2026-09-18 09:45:37 +0000 UTC"}
{"level":"info","ts":"2026-09-18T09:45:38.018Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"bb805859-11b5-4958-8e63-4107d65baec7","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:45:38.031Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"8e2f7e4d-d8b0-466f-9bdd-ddf306e885c6"}
{"level":"info","ts":"2026-09-18T09:45:38.032Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"099a66a8-3278-46d6-926c-c35e148b584b"}
{"level":"info","ts":"2026-09-18T09:45:38.040Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"0690ab89-a01a-43e4-9e1a-b803b9be9bbd","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:38.040Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"0690ab89-a01a-43e4-9e1a-b803b9be9bbd","rayCluster":"raycluster"}
{"level":"error","ts":"2026-09-18T09:45:38.043Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"0690ab89-a01a-43e4-9e1a-b803b9be9bbd","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:45:38.043Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"0690ab89-a01a-43e4-9e1a-b803b9be9bbd","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:38.043Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"0690ab89-a01a-43e4-9e1a-b803b9be9bbd"}
{"level":"error","ts":"2026-09-18T09:45:38.043Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"0690ab89-a01a-43e4-9e1a-b803b9be9bbd","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:38.070Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f4b22474-7990-4883-8f1d-88d779ee5f80","namespacedName":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:38.070Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f4b22474-7990-4883-8f1d-88d779ee5f80","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:38.070Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f4b22474-7990-4883-8f1d-88d779ee5f80","cluster":"raycluster"}
{"level":"error","ts":"2026-09-18T09:45:38.082Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f4b22474-7990-4883-8f1d-88d779ee5f80","error":"failed to ensure OIDC resources: failed to ensure ReferenceGrant: failed to create or update ReferenceGrant: referencegrants.gateway.networking.k8s.io \"kuberay-gateway-access\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:38.083Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f4b22474-7990-4883-8f1d-88d779ee5f80"}
{"level":"error","ts":"2026-09-18T09:45:38.083Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"f4b22474-7990-4883-8f1d-88d779ee5f80","error":"failed to ensure OIDC resources: failed to ensure ReferenceGrant: failed to create or update ReferenceGrant: referencegrants.gateway.networking.k8s.io \"kuberay-gateway-access\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:38.204Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"43b35fbd-8d13-4eb8-82da-8cb1ec7f2d0a","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:38.204Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"43b35fbd-8d13-4eb8-82da-8cb1ec7f2d0a","rayCluster":"raycluster"}
{"level":"error","ts":"2026-09-18T09:45:38.207Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"43b35fbd-8d13-4eb8-82da-8cb1ec7f2d0a","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:45:38.207Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"43b35fbd-8d13-4eb8-82da-8cb1ec7f2d0a","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:38.207Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"43b35fbd-8d13-4eb8-82da-8cb1ec7f2d0a"}
{"level":"error","ts":"2026-09-18T09:45:38.207Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"43b35fbd-8d13-4eb8-82da-8cb1ec7f2d0a","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:38.243Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"319d90f2-6d14-40bb-8a4b-dd0526999fc1","namespacedName":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:38.243Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"319d90f2-6d14-40bb-8a4b-dd0526999fc1","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:38.243Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"319d90f2-6d14-40bb-8a4b-dd0526999fc1","cluster":"raycluster"}
{"level":"error","ts":"2026-09-18T09:45:38.256Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"319d90f2-6d14-40bb-8a4b-dd0526999fc1","error":"failed to ensure OIDC resources: failed to ensure ReferenceGrant: failed to create or update ReferenceGrant: referencegrants.gateway.networking.k8s.io \"kuberay-gateway-access\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:38.256Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"319d90f2-6d14-40bb-8a4b-dd0526999fc1"}
{"level":"error","ts":"2026-09-18T09:45:38.256Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"319d90f2-6d14-40bb-8a4b-dd0526999fc1","error":"failed to ensure OIDC resources: failed to ensure ReferenceGrant: failed to create or update ReferenceGrant: referencegrants.gateway.networking.k8s.io \"kuberay-gateway-access\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:38.461Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"e494f99c-24ec-43ee-b867-632a76cb890e"}
{"level":"info","ts":"2026-09-18T09:45:38.461Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"reconcileID":"8908b1d7-7bca-47a3-9c92-57844f683858"}
{"level":"info","ts":"2026-09-18T09:45:38.461Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"1fb8fdf4-8b37-4704-a35c-744aaee9ba83","namespacedName":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:38.461Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"1fb8fdf4-8b37-4704-a35c-744aaee9ba83","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:38.461Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"1fb8fdf4-8b37-4704-a35c-744aaee9ba83","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:38.461Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"34dce93b-4985-4ab8-9e1a-85db23d6f774","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:38.461Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"34dce93b-4985-4ab8-9e1a-85db23d6f774","rayCluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:38.462Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"1fb8fdf4-8b37-4704-a35c-744aaee9ba83","namespace":"test-ns-r8v9f","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T09:45:38.462Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"1fb8fdf4-8b37-4704-a35c-744aaee9ba83","name":"kuberay-gateway-access"}
{"level":"error","ts":"2026-09-18T09:45:38.465Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"34dce93b-4985-4ab8-9e1a-85db23d6f774","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:45:38.465Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"34dce93b-4985-4ab8-9e1a-85db23d6f774","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:38.465Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"34dce93b-4985-4ab8-9e1a-85db23d6f774"}
{"level":"error","ts":"2026-09-18T09:45:38.465Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"34dce93b-4985-4ab8-9e1a-85db23d6f774","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-r8v9f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:38.468Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"1fb8fdf4-8b37-4704-a35c-744aaee9ba83","name":"test-ns-r8v9f-raycluster","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T09:45:38.474Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"1fb8fdf4-8b37-4704-a35c-744aaee9ba83","serviceAccount":"raycluster-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:45:38.482Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:38.483Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-r8v9f"}
{"level":"info","ts":"2026-09-18T09:45:38.493Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"1fb8fdf4-8b37-4704-a35c-744aaee9ba83","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:38.493Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d587ff75-4ff0-4b0a-bddf-e9fc97bb0146","namespacedName":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:38.493Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d587ff75-4ff0-4b0a-bddf-e9fc97bb0146","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:38.493Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d587ff75-4ff0-4b0a-bddf-e9fc97bb0146","cluster":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:38.493Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d587ff75-4ff0-4b0a-bddf-e9fc97bb0146","namespace":"test-ns-r8v9f","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T09:45:38.493Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d587ff75-4ff0-4b0a-bddf-e9fc97bb0146","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-09-18T09:45:38.493Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"dbad8420-9aa6-4d66-8da5-5506f5580ddd","namespace":"test-ns-r8v9f","clusterName":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:38.493Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"dbad8420-9aa6-4d66-8da5-5506f5580ddd","namespace":"test-ns-r8v9f","clusterName":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:38.494Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"9482c0b5-fcc0-4825-9e5c-6dc676ff159b"}
{"level":"error","ts":"2026-09-18T09:45:38.497Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d587ff75-4ff0-4b0a-bddf-e9fc97bb0146","error":"rayclusters.ray.io \"raycluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:196\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:137\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:38.497Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d587ff75-4ff0-4b0a-bddf-e9fc97bb0146"}
{"level":"error","ts":"2026-09-18T09:45:38.497Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"d587ff75-4ff0-4b0a-bddf-e9fc97bb0146","error":"rayclusters.ray.io \"raycluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:38.498Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"86b54d1d-4867-4645-aa91-60f360307c92","namespacedName":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:38.498Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"86b54d1d-4867-4645-aa91-60f360307c92"}
{"level":"info","ts":"2026-09-18T09:45:38.498Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"86b54d1d-4867-4645-aa91-60f360307c92","namespace":"test-ns-r8v9f","authClusters":0}
{"level":"info","ts":"2026-09-18T09:45:38.498Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"86b54d1d-4867-4645-aa91-60f360307c92","namespace":"test-ns-r8v9f"}
{"level":"info","ts":"2026-09-18T09:45:38.503Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"44cbb307-2283-42e0-9537-3466b76f5a3c","namespacedName":{"name":"raycluster","namespace":"test-ns-r8v9f"}}
{"level":"info","ts":"2026-09-18T09:45:38.503Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"44cbb307-2283-42e0-9537-3466b76f5a3c"}
{"level":"info","ts":"2026-09-18T09:45:38.503Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"44cbb307-2283-42e0-9537-3466b76f5a3c","namespace":"test-ns-r8v9f","authClusters":0}
{"level":"info","ts":"2026-09-18T09:45:38.503Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"44cbb307-2283-42e0-9537-3466b76f5a3c","namespace":"test-ns-r8v9f"}
{"level":"info","ts":"2026-09-18T09:45:38.528Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"722afb81-c32f-48cf-bc71-9ab9fa6c5190","namespace":"test-ns-r8v9f","clusterName":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:38.528Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-r8v9f"},"namespace":"test-ns-r8v9f","name":"raycluster","reconcileID":"722afb81-c32f-48cf-bc71-9ab9fa6c5190","namespace":"test-ns-r8v9f","clusterName":"raycluster"}
{"level":"info","ts":"2026-09-18T09:45:39.023Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"642b473e-6eb6-4b47-9e78-e91f91743cbc"}
{"level":"info","ts":"2026-09-18T09:45:39.023Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"642b473e-6eb6-4b47-9e78-e91f91743cbc"}
{"level":"info","ts":"2026-09-18T09:45:39.023Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"642b473e-6eb6-4b47-9e78-e91f91743cbc"}
{"level":"info","ts":"2026-09-18T09:45:39.023Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"642b473e-6eb6-4b47-9e78-e91f91743cbc","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:39.023Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"642b473e-6eb6-4b47-9e78-e91f91743cbc","Found 1 head Pod":"counter-wmhkk-head-j9258","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:39.023Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"642b473e-6eb6-4b47-9e78-e91f91743cbc","head Pod":"counter-wmhkk-head-j9258","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-wmhkk-head-j9258. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:39.023Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"642b473e-6eb6-4b47-9e78-e91f91743cbc","podName":"counter-wmhkk-head-j9258","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:45:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:45:39.023Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"642b473e-6eb6-4b47-9e78-e91f91743cbc","podName":"counter-wmhkk-head-j9258","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:45:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:45:39.023Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"642b473e-6eb6-4b47-9e78-e91f91743cbc","name":"counter-wmhkk","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:39.036Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"286b2405-89b1-4679-b75b-09d6f3869c72","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:39.037Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"5c3a7bad-18d4-4059-8c3e-b4d7722ec650","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:39.037Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"5c3a7bad-18d4-4059-8c3e-b4d7722ec650","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:39.037Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"5c3a7bad-18d4-4059-8c3e-b4d7722ec650","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:39.037Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"5c3a7bad-18d4-4059-8c3e-b4d7722ec650"}
{"level":"info","ts":"2026-09-18T09:45:39.037Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"5c3a7bad-18d4-4059-8c3e-b4d7722ec650","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:39.037Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"5c3a7bad-18d4-4059-8c3e-b4d7722ec650","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:39.037Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"5c3a7bad-18d4-4059-8c3e-b4d7722ec650","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:45:39.037Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"9d1b351e-ec1a-45ce-8302-13a429d447af","rayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:39.037Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"9d1b351e-ec1a-45ce-8302-13a429d447af","pod":"counter-wmhkk-head-j9258","ip":"10.133.0.28"}
{"level":"info","ts":"2026-09-18T09:45:39.037Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"9d1b351e-ec1a-45ce-8302-13a429d447af","rayCluster":"counter-wmhkk","podIPs":["10.133.0.28"]}
{"level":"info","ts":"2026-09-18T09:45:39.049Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"286b2405-89b1-4679-b75b-09d6f3869c72","name":"counter-wmhkk-head"}
{"level":"info","ts":"2026-09-18T09:45:39.050Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"68d62d59-a7fd-4bbe-bfda-7d2accb2a725","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:39.050Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"68d62d59-a7fd-4bbe-bfda-7d2accb2a725","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:39.050Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"68d62d59-a7fd-4bbe-bfda-7d2accb2a725","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:39.050Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"68d62d59-a7fd-4bbe-bfda-7d2accb2a725"}
{"level":"info","ts":"2026-09-18T09:45:39.050Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"68d62d59-a7fd-4bbe-bfda-7d2accb2a725","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:39.050Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"68d62d59-a7fd-4bbe-bfda-7d2accb2a725","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:39.061Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"286b2405-89b1-4679-b75b-09d6f3869c72","name":"counter-wmhkk-workers"}
{"level":"info","ts":"2026-09-18T09:45:39.061Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"286b2405-89b1-4679-b75b-09d6f3869c72","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:40.574Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-r8v9f"},"reconcileID":"7948b12f-4c99-4b44-9d51-c2637e5108ff"}
{"level":"info","ts":"2026-09-18T09:45:41.006Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"874745c1-c6ea-4242-9ed7-81795297a84d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:41.006Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"874745c1-c6ea-4242-9ed7-81795297a84d","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:41.006Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"874745c1-c6ea-4242-9ed7-81795297a84d","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:41.006Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"874745c1-c6ea-4242-9ed7-81795297a84d"}
{"level":"info","ts":"2026-09-18T09:45:41.006Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"874745c1-c6ea-4242-9ed7-81795297a84d","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:41.006Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"874745c1-c6ea-4242-9ed7-81795297a84d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:41.018Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter","namespace":"test-ns-r8v9f"},"reconcileID":"27729d95-4a54-48dd-a9cd-e3b54d981808"}
{"level":"info","ts":"2026-09-18T09:45:41.038Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6405cf57-bd52-44f4-978a-a1e0c0d39592"}
{"level":"info","ts":"2026-09-18T09:45:41.038Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6405cf57-bd52-44f4-978a-a1e0c0d39592"}
{"level":"info","ts":"2026-09-18T09:45:41.038Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6405cf57-bd52-44f4-978a-a1e0c0d39592"}
{"level":"info","ts":"2026-09-18T09:45:41.038Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6405cf57-bd52-44f4-978a-a1e0c0d39592","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:41.038Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6405cf57-bd52-44f4-978a-a1e0c0d39592","Found 1 head Pod":"counter-wmhkk-head-j9258","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:41.038Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6405cf57-bd52-44f4-978a-a1e0c0d39592","head Pod":"counter-wmhkk-head-j9258","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-wmhkk-head-j9258. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:41.038Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6405cf57-bd52-44f4-978a-a1e0c0d39592","podName":"counter-wmhkk-head-j9258","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:45:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:45:41.038Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6405cf57-bd52-44f4-978a-a1e0c0d39592","podName":"counter-wmhkk-head-j9258","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:45:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:45:41.038Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6405cf57-bd52-44f4-978a-a1e0c0d39592","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:45:41.038Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"6405cf57-bd52-44f4-978a-a1e0c0d39592","seconds":300}
{"level":"info","ts":"2026-09-18T09:45:44.007Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"3d69a809-2223-4b04-b2d5-af4b96647213","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:44.007Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"3d69a809-2223-4b04-b2d5-af4b96647213","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:44.007Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"3d69a809-2223-4b04-b2d5-af4b96647213","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:44.007Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"3d69a809-2223-4b04-b2d5-af4b96647213"}
{"level":"info","ts":"2026-09-18T09:45:44.007Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"3d69a809-2223-4b04-b2d5-af4b96647213","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:44.007Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"3d69a809-2223-4b04-b2d5-af4b96647213","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:47.007Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"54f3d06b-d549-4bad-ac8e-c7016c0e868c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:47.007Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"54f3d06b-d549-4bad-ac8e-c7016c0e868c","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:47.007Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"54f3d06b-d549-4bad-ac8e-c7016c0e868c","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:47.007Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"54f3d06b-d549-4bad-ac8e-c7016c0e868c"}
{"level":"info","ts":"2026-09-18T09:45:47.007Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"54f3d06b-d549-4bad-ac8e-c7016c0e868c","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:47.007Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"54f3d06b-d549-4bad-ac8e-c7016c0e868c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:50.008Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"8da274d8-4df4-405a-bc8c-193ab03da435","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:50.008Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"8da274d8-4df4-405a-bc8c-193ab03da435","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:50.008Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"8da274d8-4df4-405a-bc8c-193ab03da435","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:50.008Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"8da274d8-4df4-405a-bc8c-193ab03da435"}
{"level":"info","ts":"2026-09-18T09:45:50.008Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"8da274d8-4df4-405a-bc8c-193ab03da435","RayCluster":"counter-wmhkk","State":""}
{"level":"info","ts":"2026-09-18T09:45:50.008Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"8da274d8-4df4-405a-bc8c-193ab03da435","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:50.066Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"267ea2ad-39a0-4cb5-b390-8f0a895ec5c3"}
{"level":"info","ts":"2026-09-18T09:45:50.066Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"267ea2ad-39a0-4cb5-b390-8f0a895ec5c3"}
{"level":"info","ts":"2026-09-18T09:45:50.066Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"267ea2ad-39a0-4cb5-b390-8f0a895ec5c3"}
{"level":"info","ts":"2026-09-18T09:45:50.066Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"267ea2ad-39a0-4cb5-b390-8f0a895ec5c3","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:50.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"267ea2ad-39a0-4cb5-b390-8f0a895ec5c3","Found 1 head Pod":"counter-wmhkk-head-j9258","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:50.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"267ea2ad-39a0-4cb5-b390-8f0a895ec5c3","head Pod":"counter-wmhkk-head-j9258","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-wmhkk-head-j9258. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:50.066Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"267ea2ad-39a0-4cb5-b390-8f0a895ec5c3","name":"counter-wmhkk","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:50Z","stateTransitionTimes":{"ready":"2026-09-18T09:45:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:50.079Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"9eb7998e-99ed-4446-a32a-1815268d4613","rayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:50.079Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"95665bc6-26b8-42e4-9980-b37c6cd09f47","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:50.079Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"9eb7998e-99ed-4446-a32a-1815268d4613","pod":"counter-wmhkk-head-j9258","ip":"10.133.0.28"}
{"level":"info","ts":"2026-09-18T09:45:50.079Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"9eb7998e-99ed-4446-a32a-1815268d4613","rayCluster":"counter-wmhkk","podIPs":["10.133.0.28"]}
{"level":"info","ts":"2026-09-18T09:45:50.080Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"402e3b2a-2fb2-4153-a72b-8dba8e041238","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:50.080Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"402e3b2a-2fb2-4153-a72b-8dba8e041238","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:50.080Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"402e3b2a-2fb2-4153-a72b-8dba8e041238","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:50.080Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"402e3b2a-2fb2-4153-a72b-8dba8e041238"}
{"level":"info","ts":"2026-09-18T09:45:50.080Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"402e3b2a-2fb2-4153-a72b-8dba8e041238","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","dashboardURL":"counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:50.080Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"402e3b2a-2fb2-4153-a72b-8dba8e041238","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:45:50.095Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"95665bc6-26b8-42e4-9980-b37c6cd09f47","name":"counter-wmhkk-head"}
{"level":"info","ts":"2026-09-18T09:45:50.097Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9c8f4f91-7b93-4910-8ca6-13ee6a85d23f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:50.097Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9c8f4f91-7b93-4910-8ca6-13ee6a85d23f","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:50.097Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9c8f4f91-7b93-4910-8ca6-13ee6a85d23f","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:50.097Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9c8f4f91-7b93-4910-8ca6-13ee6a85d23f"}
{"level":"info","ts":"2026-09-18T09:45:50.097Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9c8f4f91-7b93-4910-8ca6-13ee6a85d23f","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:45:50.108Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"95665bc6-26b8-42e4-9980-b37c6cd09f47","name":"counter-wmhkk-workers"}
{"level":"info","ts":"2026-09-18T09:45:50.108Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"95665bc6-26b8-42e4-9980-b37c6cd09f47","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:50.119Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9c8f4f91-7b93-4910-8ca6-13ee6a85d23f","JobId":"counter-5cmds"}
{"level":"info","ts":"2026-09-18T09:45:52.083Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"832c3d2e-8047-4333-ac6a-cab04d2b2cf6"}
{"level":"info","ts":"2026-09-18T09:45:52.083Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"832c3d2e-8047-4333-ac6a-cab04d2b2cf6"}
{"level":"info","ts":"2026-09-18T09:45:52.083Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"832c3d2e-8047-4333-ac6a-cab04d2b2cf6"}
{"level":"info","ts":"2026-09-18T09:45:52.083Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"832c3d2e-8047-4333-ac6a-cab04d2b2cf6","1 head service found":"counter-wmhkk-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:52.083Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"832c3d2e-8047-4333-ac6a-cab04d2b2cf6","Found 1 head Pod":"counter-wmhkk-head-j9258","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:45:52.083Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"832c3d2e-8047-4333-ac6a-cab04d2b2cf6","head Pod":"counter-wmhkk-head-j9258","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-wmhkk-head-j9258. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:45:52.083Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"832c3d2e-8047-4333-ac6a-cab04d2b2cf6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:45:52.083Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"832c3d2e-8047-4333-ac6a-cab04d2b2cf6","seconds":300}
{"level":"error","ts":"2026-09-18T09:45:52.120Z","logger":"controllers.RayJob","msg":"Failed to submit the Ray job","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9c8f4f91-7b93-4910-8ca6-13ee6a85d23f","JobId":"counter-5cmds","error":"Post \"http://counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265/api/jobs/\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:52.120Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9c8f4f91-7b93-4910-8ca6-13ee6a85d23f"}
{"level":"error","ts":"2026-09-18T09:45:52.120Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9c8f4f91-7b93-4910-8ca6-13ee6a85d23f","error":"Post \"http://counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265/api/jobs/\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:52.120Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"6f5efcd4-7380-4730-bf75-ea03610b2ed8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:52.120Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"6f5efcd4-7380-4730-bf75-ea03610b2ed8","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:52.120Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"6f5efcd4-7380-4730-bf75-ea03610b2ed8","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:52.120Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"6f5efcd4-7380-4730-bf75-ea03610b2ed8"}
{"level":"info","ts":"2026-09-18T09:45:52.120Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"6f5efcd4-7380-4730-bf75-ea03610b2ed8","rayCluster":"counter-wmhkk","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:45:52.178Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"6f5efcd4-7380-4730-bf75-ea03610b2ed8","JobId":"counter-5cmds"}
{"level":"error","ts":"2026-09-18T09:45:53.316Z","logger":"controllers.RayJob","msg":"Failed to submit the Ray job","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"6f5efcd4-7380-4730-bf75-ea03610b2ed8","JobId":"counter-5cmds","error":"submission ID 'counter-5cmds' already used, Please use a different submission_id","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:53.316Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"6f5efcd4-7380-4730-bf75-ea03610b2ed8"}
{"level":"error","ts":"2026-09-18T09:45:53.316Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"6f5efcd4-7380-4730-bf75-ea03610b2ed8","error":"submission ID 'counter-5cmds' already used, Please use a different submission_id","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:53.316Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"55ba4664-deba-4746-8e1b-7e140f58a78e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:53.316Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"55ba4664-deba-4746-8e1b-7e140f58a78e","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:53.316Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"55ba4664-deba-4746-8e1b-7e140f58a78e","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:53.317Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"55ba4664-deba-4746-8e1b-7e140f58a78e"}
{"level":"info","ts":"2026-09-18T09:45:53.317Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"55ba4664-deba-4746-8e1b-7e140f58a78e","rayCluster":"counter-wmhkk","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:45:53.319Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"55ba4664-deba-4746-8e1b-7e140f58a78e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","dashboardURL":"counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:53Z"},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","dashboardURL":"counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:50Z","stateTransitionTimes":{"ready":"2026-09-18T09:45:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:53.319Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"55ba4664-deba-4746-8e1b-7e140f58a78e","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:45:53.334Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9018a94e-6493-41de-b92e-8568eb5fd317","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:53.334Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9018a94e-6493-41de-b92e-8568eb5fd317","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:53.334Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9018a94e-6493-41de-b92e-8568eb5fd317","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:53.334Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9018a94e-6493-41de-b92e-8568eb5fd317"}
{"level":"info","ts":"2026-09-18T09:45:53.334Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9018a94e-6493-41de-b92e-8568eb5fd317","rayCluster":"counter-wmhkk","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:45:53.337Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9018a94e-6493-41de-b92e-8568eb5fd317","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","dashboardURL":"counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:53Z"},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","dashboardURL":"counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:50Z","stateTransitionTimes":{"ready":"2026-09-18T09:45:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:53.337Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9018a94e-6493-41de-b92e-8568eb5fd317","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:45:53.344Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9018a94e-6493-41de-b92e-8568eb5fd317","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T09:45:53.344Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9018a94e-6493-41de-b92e-8568eb5fd317"}
{"level":"error","ts":"2026-09-18T09:45:53.344Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"9018a94e-6493-41de-b92e-8568eb5fd317","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:53.344Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"5a9c8306-9875-4f56-8a67-3e7bbffe95a7","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:53.344Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"5a9c8306-9875-4f56-8a67-3e7bbffe95a7","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:53.344Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"5a9c8306-9875-4f56-8a67-3e7bbffe95a7","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:53.344Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"5a9c8306-9875-4f56-8a67-3e7bbffe95a7"}
{"level":"info","ts":"2026-09-18T09:45:53.344Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"5a9c8306-9875-4f56-8a67-3e7bbffe95a7","rayCluster":"counter-wmhkk","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:45:53.346Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"5a9c8306-9875-4f56-8a67-3e7bbffe95a7","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:53Z"},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","dashboardURL":"counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:50Z","stateTransitionTimes":{"ready":"2026-09-18T09:45:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:53Z"},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","dashboardURL":"counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:50Z","stateTransitionTimes":{"ready":"2026-09-18T09:45:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:53.350Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"38f8a1b1-45e4-49c8-9076-400ccfb5959c","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:53.350Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"38f8a1b1-45e4-49c8-9076-400ccfb5959c","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:53.350Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"38f8a1b1-45e4-49c8-9076-400ccfb5959c","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:53.350Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"38f8a1b1-45e4-49c8-9076-400ccfb5959c"}
{"level":"info","ts":"2026-09-18T09:45:53.350Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"38f8a1b1-45e4-49c8-9076-400ccfb5959c","rayCluster":"counter-wmhkk","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:45:53.352Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"38f8a1b1-45e4-49c8-9076-400ccfb5959c","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:53Z"},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","dashboardURL":"counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:50Z","stateTransitionTimes":{"ready":"2026-09-18T09:45:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:53Z"},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","dashboardURL":"counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:50Z","stateTransitionTimes":{"ready":"2026-09-18T09:45:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:56.353Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b1c5d64c-c926-4ab1-b00d-ed3c2e35b547","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:56.353Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b1c5d64c-c926-4ab1-b00d-ed3c2e35b547","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:56.353Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b1c5d64c-c926-4ab1-b00d-ed3c2e35b547","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:56.353Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b1c5d64c-c926-4ab1-b00d-ed3c2e35b547"}
{"level":"info","ts":"2026-09-18T09:45:56.353Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b1c5d64c-c926-4ab1-b00d-ed3c2e35b547","rayCluster":"counter-wmhkk","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:45:56.356Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b1c5d64c-c926-4ab1-b00d-ed3c2e35b547","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:53Z"},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","dashboardURL":"counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:50Z","stateTransitionTimes":{"ready":"2026-09-18T09:45:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:53Z"},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","dashboardURL":"counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:50Z","stateTransitionTimes":{"ready":"2026-09-18T09:45:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:56.356Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b1c5d64c-c926-4ab1-b00d-ed3c2e35b547","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:45:56.373Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"05fe3dbd-b5a8-4da1-872c-1854615b2844","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:56.373Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"05fe3dbd-b5a8-4da1-872c-1854615b2844","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:56.373Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"05fe3dbd-b5a8-4da1-872c-1854615b2844","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:56.373Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"05fe3dbd-b5a8-4da1-872c-1854615b2844"}
{"level":"info","ts":"2026-09-18T09:45:56.373Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"05fe3dbd-b5a8-4da1-872c-1854615b2844","rayCluster":"counter-wmhkk","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:45:56.376Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"05fe3dbd-b5a8-4da1-872c-1854615b2844","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:53Z"},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","dashboardURL":"counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:50Z","stateTransitionTimes":{"ready":"2026-09-18T09:45:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:53Z"},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","dashboardURL":"counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:50Z","stateTransitionTimes":{"ready":"2026-09-18T09:45:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:57.580Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"17cc87b0-fca6-4620-83c8-29319d717f8b","rayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:57.580Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"17cc87b0-fca6-4620-83c8-29319d717f8b","pod":"counter-wmhkk-head-j9258","ip":"10.133.0.28"}
{"level":"info","ts":"2026-09-18T09:45:57.580Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"17cc87b0-fca6-4620-83c8-29319d717f8b","rayCluster":"counter-wmhkk","podIPs":["10.133.0.28"]}
{"level":"info","ts":"2026-09-18T09:45:59.372Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"99a4b3d2-de45-46a8-81f8-a212d0490535","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:59.372Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"99a4b3d2-de45-46a8-81f8-a212d0490535","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:59.372Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"99a4b3d2-de45-46a8-81f8-a212d0490535","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:59.372Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"99a4b3d2-de45-46a8-81f8-a212d0490535"}
{"level":"info","ts":"2026-09-18T09:45:59.373Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"99a4b3d2-de45-46a8-81f8-a212d0490535","rayCluster":"counter-wmhkk","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:45:59.375Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"99a4b3d2-de45-46a8-81f8-a212d0490535","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:53Z"},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","dashboardURL":"counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:45:27Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:50Z","stateTransitionTimes":{"ready":"2026-09-18T09:45:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:45:53Z","endTime":"2026-09-18T09:45:57Z"},"jobId":"counter-5cmds","rayClusterName":"counter-wmhkk","dashboardURL":"counter-wmhkk-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-09-18T09:45:27Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:50Z","stateTransitionTimes":{"ready":"2026-09-18T09:45:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-wmhkk-head-j9258","serviceName":"counter-wmhkk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:45:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:27Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:27Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:59.375Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"99a4b3d2-de45-46a8-81f8-a212d0490535","old JobStatus":"RUNNING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-09-18T09:45:59.391Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"963646e8-6304-4a03-b8d9-c1ecb0c8f20a","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:59.391Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"963646e8-6304-4a03-b8d9-c1ecb0c8f20a","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:45:59 +0000 UTC","Now":"2026-09-18T09:45:59.391Z","ShutdownTime":"2026-09-18T09:45:59.000Z"}
{"level":"info","ts":"2026-09-18T09:45:59.400Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"963646e8-6304-4a03-b8d9-c1ecb0c8f20a","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:59.400Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"963646e8-6304-4a03-b8d9-c1ecb0c8f20a","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T09:45:59.400Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"963646e8-6304-4a03-b8d9-c1ecb0c8f20a","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:59.401Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"3973967f-d957-4b85-80c4-d6edbd3fe28c"}
{"level":"info","ts":"2026-09-18T09:45:59.401Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"986ea0c8-5f93-4db7-8c34-d12300d6601a","rayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:59.401Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"reconcileID":"ca64b10e-8612-4765-b3ec-94175eda5454"}
{"level":"info","ts":"2026-09-18T09:45:59.401Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"986ea0c8-5f93-4db7-8c34-d12300d6601a","pod":"counter-wmhkk-head-j9258","ip":"10.133.0.28"}
{"level":"info","ts":"2026-09-18T09:45:59.401Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"986ea0c8-5f93-4db7-8c34-d12300d6601a","rayCluster":"counter-wmhkk","podIPs":["10.133.0.28"]}
{"level":"info","ts":"2026-09-18T09:45:59.401Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"b85b9591-513c-4df8-8085-f917bc99dfb5","namespacedName":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:59.401Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"b85b9591-513c-4df8-8085-f917bc99dfb5","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:59.401Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"b85b9591-513c-4df8-8085-f917bc99dfb5","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:59.401Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"f6300aa6-2b74-4049-bc3b-bcede54991ca","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:59.401Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"b85b9591-513c-4df8-8085-f917bc99dfb5","namespace":"test-ns-q2lj2","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T09:45:59.401Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"f6300aa6-2b74-4049-bc3b-bcede54991ca","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:45:59 +0000 UTC","Now":"2026-09-18T09:45:59.401Z","ShutdownTime":"2026-09-18T09:45:59.000Z"}
{"level":"info","ts":"2026-09-18T09:45:59.401Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"f6300aa6-2b74-4049-bc3b-bcede54991ca","RayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:59.401Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"f6300aa6-2b74-4049-bc3b-bcede54991ca","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T09:45:59.401Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"f6300aa6-2b74-4049-bc3b-bcede54991ca","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:59.407Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"b85b9591-513c-4df8-8085-f917bc99dfb5","name":"kuberay-gateway-access","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:45:59.413Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"b85b9591-513c-4df8-8085-f917bc99dfb5","name":"test-ns-q2lj2-counter-wmhkk","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T09:45:59.418Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"b85b9591-513c-4df8-8085-f917bc99dfb5","configMap":"kube-rbac-proxy-config-counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:59.424Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"b85b9591-513c-4df8-8085-f917bc99dfb5","serviceAccount":"counter-wmhkk-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:45:59.432Z","logger":"raycluster-resource","msg":"default","name":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:59.432Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-wmhkk","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:45:59.445Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"b85b9591-513c-4df8-8085-f917bc99dfb5","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:59.445Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c957cc6e-e7d8-42e1-83c6-cea275ae8989","namespacedName":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:59.445Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c957cc6e-e7d8-42e1-83c6-cea275ae8989","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:59.445Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c957cc6e-e7d8-42e1-83c6-cea275ae8989","cluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:59.445Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c957cc6e-e7d8-42e1-83c6-cea275ae8989","namespace":"test-ns-q2lj2","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T09:45:59.445Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c957cc6e-e7d8-42e1-83c6-cea275ae8989","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-09-18T09:45:59.445Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"8ae3cfcf-aac4-4b8e-90e3-d12306c979b8","namespace":"test-ns-q2lj2","clusterName":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:59.445Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"a3d3c45a-7464-4d83-bb5d-8c8a28901bb3"}
{"level":"info","ts":"2026-09-18T09:45:59.445Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"ba07a0ef-ccdc-43e4-b7e5-0d0053e67b82","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:59.445Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"ba07a0ef-ccdc-43e4-b7e5-0d0053e67b82","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:45:59 +0000 UTC","Now":"2026-09-18T09:45:59.445Z","ShutdownTime":"2026-09-18T09:45:59.000Z"}
{"level":"info","ts":"2026-09-18T09:45:59.445Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"ba07a0ef-ccdc-43e4-b7e5-0d0053e67b82","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:59.445Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"ba07a0ef-ccdc-43e4-b7e5-0d0053e67b82","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T09:45:59.445Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"ba07a0ef-ccdc-43e4-b7e5-0d0053e67b82","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-wmhkk"}
{"level":"error","ts":"2026-09-18T09:45:59.466Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c957cc6e-e7d8-42e1-83c6-cea275ae8989","error":"rayclusters.ray.io \"counter-wmhkk\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:196\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:137\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:59.466Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c957cc6e-e7d8-42e1-83c6-cea275ae8989"}
{"level":"error","ts":"2026-09-18T09:45:59.466Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"c957cc6e-e7d8-42e1-83c6-cea275ae8989","error":"rayclusters.ray.io \"counter-wmhkk\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:59.466Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"012eadbf-a295-4dc2-af1e-f26e57dc0a97","namespacedName":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:59.466Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"012eadbf-a295-4dc2-af1e-f26e57dc0a97"}
{"level":"info","ts":"2026-09-18T09:45:59.466Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"012eadbf-a295-4dc2-af1e-f26e57dc0a97","namespace":"test-ns-q2lj2","authClusters":0}
{"level":"info","ts":"2026-09-18T09:45:59.466Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"012eadbf-a295-4dc2-af1e-f26e57dc0a97","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:45:59.467Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"6336a412-45a7-4197-84eb-415ee25f9c2e"}
{"level":"error","ts":"2026-09-18T09:45:59.471Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"8ae3cfcf-aac4-4b8e-90e3-d12306c979b8","issuer":"ray-selfsigned-issuer-counter-wmhkk","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-wmhkk\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:59.471Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"80c89c3e-ed9f-4467-beb5-ce58ff0262d3","namespacedName":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:59.471Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"80c89c3e-ed9f-4467-beb5-ce58ff0262d3"}
{"level":"info","ts":"2026-09-18T09:45:59.471Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"80c89c3e-ed9f-4467-beb5-ce58ff0262d3","namespace":"test-ns-q2lj2","authClusters":0}
{"level":"info","ts":"2026-09-18T09:45:59.471Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"80c89c3e-ed9f-4467-beb5-ce58ff0262d3","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:45:59.493Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"8ae3cfcf-aac4-4b8e-90e3-d12306c979b8","errorCount":1}
{"level":"info","ts":"2026-09-18T09:45:59.493Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"8ae3cfcf-aac4-4b8e-90e3-d12306c979b8"}
{"level":"error","ts":"2026-09-18T09:45:59.493Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"8ae3cfcf-aac4-4b8e-90e3-d12306c979b8","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-wmhkk\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:59.498Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"316220fe-1aa3-44b1-809a-fddf7497912c","namespace":"test-ns-q2lj2","clusterName":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:59.498Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-wmhkk","reconcileID":"316220fe-1aa3-44b1-809a-fddf7497912c","namespace":"test-ns-q2lj2","clusterName":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:45:59.548Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6001196a-d835-45f4-b03b-8a830dcde006","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:59.548Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6001196a-d835-45f4-b03b-8a830dcde006","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:45:59.560Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6001196a-d835-45f4-b03b-8a830dcde006"}
{"level":"info","ts":"2026-09-18T09:45:59.560Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6001196a-d835-45f4-b03b-8a830dcde006","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:45:59.560Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6001196a-d835-45f4-b03b-8a830dcde006","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:45:59.560Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6001196a-d835-45f4-b03b-8a830dcde006","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:45:59.570Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6988b7af-8c51-426d-934f-23d61bf0ce51","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:59.570Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6988b7af-8c51-426d-934f-23d61bf0ce51","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.570Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6988b7af-8c51-426d-934f-23d61bf0ce51","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:59.570Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6988b7af-8c51-426d-934f-23d61bf0ce51","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:59.575Z","logger":"raycluster-resource","msg":"default","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.575Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-bcffx","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:45:59.587Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6988b7af-8c51-426d-934f-23d61bf0ce51","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:59.587Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"1a2f4db4-fb38-457a-9e00-33661e69f130","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.587Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6988b7af-8c51-426d-934f-23d61bf0ce51"}
{"level":"info","ts":"2026-09-18T09:45:59.587Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6988b7af-8c51-426d-934f-23d61bf0ce51","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:45:59.587Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6988b7af-8c51-426d-934f-23d61bf0ce51","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:45:59.588Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"43bf8ea2-ae0b-4e8b-b532-210dcaae1e4a","namespacedName":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:59.588Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"43bf8ea2-ae0b-4e8b-b532-210dcaae1e4a","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.588Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"43bf8ea2-ae0b-4e8b-b532-210dcaae1e4a","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.588Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"43bf8ea2-ae0b-4e8b-b532-210dcaae1e4a","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.588Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"4977a736-0c37-40e4-a505-1b39fc45e5bf"}
{"level":"info","ts":"2026-09-18T09:45:59.588Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"4977a736-0c37-40e4-a505-1b39fc45e5bf"}
{"level":"info","ts":"2026-09-18T09:45:59.588Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"4977a736-0c37-40e4-a505-1b39fc45e5bf"}
{"level":"info","ts":"2026-09-18T09:45:59.588Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"2b1c9939-bbe7-4307-88a8-94c2fb676035","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:59.588Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"2b1c9939-bbe7-4307-88a8-94c2fb676035","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.588Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"2b1c9939-bbe7-4307-88a8-94c2fb676035","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:59.588Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c1e468a4-522f-40ed-8be0-c6de7f120745","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.588Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c1e468a4-522f-40ed-8be0-c6de7f120745","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.588Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"2b1c9939-bbe7-4307-88a8-94c2fb676035"}
{"level":"info","ts":"2026-09-18T09:45:59.588Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"2b1c9939-bbe7-4307-88a8-94c2fb676035","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:45:59.588Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"2b1c9939-bbe7-4307-88a8-94c2fb676035","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:45:59.593Z","logger":"raycluster-resource","msg":"default","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.593Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-bcffx","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:45:59.597Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"4977a736-0c37-40e4-a505-1b39fc45e5bf","name":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:59.597Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"4977a736-0c37-40e4-a505-1b39fc45e5bf","error":"head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"error","ts":"2026-09-18T09:45:59.597Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"4977a736-0c37-40e4-a505-1b39fc45e5bf","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:59.597Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"4977a736-0c37-40e4-a505-1b39fc45e5bf","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"info","ts":"2026-09-18T09:45:59.597Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"4977a736-0c37-40e4-a505-1b39fc45e5bf","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:59.597Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"4977a736-0c37-40e4-a505-1b39fc45e5bf","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:59.597Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"4977a736-0c37-40e4-a505-1b39fc45e5bf","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:59.597Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"4977a736-0c37-40e4-a505-1b39fc45e5bf","name":"fail-bcffx","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:59.603Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c1e468a4-522f-40ed-8be0-c6de7f120745","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.603Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c1e468a4-522f-40ed-8be0-c6de7f120745","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.666Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"1a2f4db4-fb38-457a-9e00-33661e69f130","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.666Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"47689186-609f-439f-bf20-0dd28a7700ca","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.667Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"4977a736-0c37-40e4-a505-1b39fc45e5bf","name":"fail-bcffx","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-bcffx\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-q2lj2","name":"fail-bcffx"}}
{"level":"info","ts":"2026-09-18T09:45:59.667Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"4977a736-0c37-40e4-a505-1b39fc45e5bf"}
{"level":"error","ts":"2026-09-18T09:45:59.667Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"4977a736-0c37-40e4-a505-1b39fc45e5bf","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:59.667Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9ae29ac2-dbda-4bb4-b854-3ffcbc76a8f8"}
{"level":"info","ts":"2026-09-18T09:45:59.667Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9ae29ac2-dbda-4bb4-b854-3ffcbc76a8f8"}
{"level":"info","ts":"2026-09-18T09:45:59.667Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9ae29ac2-dbda-4bb4-b854-3ffcbc76a8f8"}
{"level":"info","ts":"2026-09-18T09:45:59.667Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9ae29ac2-dbda-4bb4-b854-3ffcbc76a8f8","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:59.667Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9ae29ac2-dbda-4bb4-b854-3ffcbc76a8f8","error":"head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"error","ts":"2026-09-18T09:45:59.667Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9ae29ac2-dbda-4bb4-b854-3ffcbc76a8f8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:59.667Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9ae29ac2-dbda-4bb4-b854-3ffcbc76a8f8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"info","ts":"2026-09-18T09:45:59.667Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c1e468a4-522f-40ed-8be0-c6de7f120745","rayCluster":"fail-bcffx","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T09:45:59.667Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c1e468a4-522f-40ed-8be0-c6de7f120745","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.667Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9ae29ac2-dbda-4bb4-b854-3ffcbc76a8f8","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:59.668Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9ae29ac2-dbda-4bb4-b854-3ffcbc76a8f8","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:59.668Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9ae29ac2-dbda-4bb4-b854-3ffcbc76a8f8","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:59.668Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9ae29ac2-dbda-4bb4-b854-3ffcbc76a8f8","name":"fail-bcffx","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:59.668Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"3fbe751b-7d0d-4fc6-ae81-b0e04692725b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:59.668Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"3fbe751b-7d0d-4fc6-ae81-b0e04692725b","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.668Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"3fbe751b-7d0d-4fc6-ae81-b0e04692725b","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:59.668Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"3fbe751b-7d0d-4fc6-ae81-b0e04692725b"}
{"level":"info","ts":"2026-09-18T09:45:59.668Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"3fbe751b-7d0d-4fc6-ae81-b0e04692725b","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:45:59.668Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"3fbe751b-7d0d-4fc6-ae81-b0e04692725b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:45:59.674Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"43bf8ea2-ae0b-4e8b-b532-210dcaae1e4a","name":"fail-bcffx-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:45:59.675Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9ae29ac2-dbda-4bb4-b854-3ffcbc76a8f8","name":"fail-bcffx","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-bcffx\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-q2lj2","name":"fail-bcffx"}}
{"level":"info","ts":"2026-09-18T09:45:59.675Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9ae29ac2-dbda-4bb4-b854-3ffcbc76a8f8"}
{"level":"error","ts":"2026-09-18T09:45:59.675Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9ae29ac2-dbda-4bb4-b854-3ffcbc76a8f8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:59.675Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5cd9bdbe-2bd6-44a7-9b1a-cb9a251adc2b"}
{"level":"info","ts":"2026-09-18T09:45:59.675Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5cd9bdbe-2bd6-44a7-9b1a-cb9a251adc2b"}
{"level":"info","ts":"2026-09-18T09:45:59.675Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5cd9bdbe-2bd6-44a7-9b1a-cb9a251adc2b"}
{"level":"info","ts":"2026-09-18T09:45:59.675Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5cd9bdbe-2bd6-44a7-9b1a-cb9a251adc2b","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:59.675Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5cd9bdbe-2bd6-44a7-9b1a-cb9a251adc2b","error":"head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"error","ts":"2026-09-18T09:45:59.675Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5cd9bdbe-2bd6-44a7-9b1a-cb9a251adc2b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:59.675Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5cd9bdbe-2bd6-44a7-9b1a-cb9a251adc2b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"info","ts":"2026-09-18T09:45:59.675Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5cd9bdbe-2bd6-44a7-9b1a-cb9a251adc2b","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:59.675Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5cd9bdbe-2bd6-44a7-9b1a-cb9a251adc2b","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:59.675Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5cd9bdbe-2bd6-44a7-9b1a-cb9a251adc2b","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:59.675Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5cd9bdbe-2bd6-44a7-9b1a-cb9a251adc2b","name":"fail-bcffx","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:45:59.677Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c1e468a4-522f-40ed-8be0-c6de7f120745","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.681Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"47689186-609f-439f-bf20-0dd28a7700ca","name":"fail-bcffx-head"}
{"level":"info","ts":"2026-09-18T09:45:59.683Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"43bf8ea2-ae0b-4e8b-b532-210dcaae1e4a","name":"kuberay-gateway-access","namespace":"test-ns-q2lj2","operation":"created"}
{"level":"info","ts":"2026-09-18T09:45:59.689Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"09c4eea8-6887-4644-9045-6e7114340e8a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:59.689Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"09c4eea8-6887-4644-9045-6e7114340e8a","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.689Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"09c4eea8-6887-4644-9045-6e7114340e8a","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:59.690Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"09c4eea8-6887-4644-9045-6e7114340e8a"}
{"level":"info","ts":"2026-09-18T09:45:59.690Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"09c4eea8-6887-4644-9045-6e7114340e8a","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:45:59.690Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"09c4eea8-6887-4644-9045-6e7114340e8a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:59.690Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"09c4eea8-6887-4644-9045-6e7114340e8a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:45:59.766Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5cd9bdbe-2bd6-44a7-9b1a-cb9a251adc2b"}
{"level":"error","ts":"2026-09-18T09:45:59.766Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5cd9bdbe-2bd6-44a7-9b1a-cb9a251adc2b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:59.766Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cbe470e9-43bb-43e6-93ff-21976b96829c"}
{"level":"info","ts":"2026-09-18T09:45:59.766Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cbe470e9-43bb-43e6-93ff-21976b96829c"}
{"level":"info","ts":"2026-09-18T09:45:59.766Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cbe470e9-43bb-43e6-93ff-21976b96829c"}
{"level":"info","ts":"2026-09-18T09:45:59.767Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cbe470e9-43bb-43e6-93ff-21976b96829c","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:59.767Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cbe470e9-43bb-43e6-93ff-21976b96829c","error":"head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"error","ts":"2026-09-18T09:45:59.767Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cbe470e9-43bb-43e6-93ff-21976b96829c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:59.767Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cbe470e9-43bb-43e6-93ff-21976b96829c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"info","ts":"2026-09-18T09:45:59.767Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cbe470e9-43bb-43e6-93ff-21976b96829c","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:59.767Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cbe470e9-43bb-43e6-93ff-21976b96829c","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:59.767Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cbe470e9-43bb-43e6-93ff-21976b96829c","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:59.767Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cbe470e9-43bb-43e6-93ff-21976b96829c"}
{"level":"error","ts":"2026-09-18T09:45:59.767Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cbe470e9-43bb-43e6-93ff-21976b96829c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:59.767Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"43bf8ea2-ae0b-4e8b-b532-210dcaae1e4a","name":"test-ns-q2lj2-fail-bcffx","operation":"created"}
{"level":"info","ts":"2026-09-18T09:45:59.768Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c1e468a4-522f-40ed-8be0-c6de7f120745","rayCluster":"fail-bcffx","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:45:59.768Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"96286504-11a1-4286-b3fb-53c5b9fead55","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:45:59.768Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"96286504-11a1-4286-b3fb-53c5b9fead55","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.768Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"96286504-11a1-4286-b3fb-53c5b9fead55","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:59.768Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"96286504-11a1-4286-b3fb-53c5b9fead55"}
{"level":"info","ts":"2026-09-18T09:45:59.768Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"96286504-11a1-4286-b3fb-53c5b9fead55","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:45:59.768Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"96286504-11a1-4286-b3fb-53c5b9fead55","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:45:59.775Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"43bf8ea2-ae0b-4e8b-b532-210dcaae1e4a","name":"kube-rbac-proxy-config-fail-bcffx","operation":"created"}
{"level":"info","ts":"2026-09-18T09:45:59.775Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"47689186-609f-439f-bf20-0dd28a7700ca","name":"fail-bcffx-workers"}
{"level":"info","ts":"2026-09-18T09:45:59.775Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"47689186-609f-439f-bf20-0dd28a7700ca","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.775Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"541e0b26-089d-48b1-9081-20f3d1d66d92","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.778Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c1e468a4-522f-40ed-8be0-c6de7f120745","rayCluster":"fail-bcffx","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:45:59.778Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c1e468a4-522f-40ed-8be0-c6de7f120745"}
{"level":"info","ts":"2026-09-18T09:45:59.966Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c5861b94-204f-4964-a075-26c977a7fba3"}
{"level":"info","ts":"2026-09-18T09:45:59.966Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c5861b94-204f-4964-a075-26c977a7fba3"}
{"level":"info","ts":"2026-09-18T09:45:59.966Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c5861b94-204f-4964-a075-26c977a7fba3"}
{"level":"info","ts":"2026-09-18T09:45:59.966Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c5861b94-204f-4964-a075-26c977a7fba3","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:45:59.966Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c5861b94-204f-4964-a075-26c977a7fba3","error":"head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"error","ts":"2026-09-18T09:45:59.966Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c5861b94-204f-4964-a075-26c977a7fba3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:59.966Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c5861b94-204f-4964-a075-26c977a7fba3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"info","ts":"2026-09-18T09:45:59.966Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c5861b94-204f-4964-a075-26c977a7fba3","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:59.966Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c5861b94-204f-4964-a075-26c977a7fba3","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:59.966Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c5861b94-204f-4964-a075-26c977a7fba3","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:45:59.966Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c5861b94-204f-4964-a075-26c977a7fba3"}
{"level":"error","ts":"2026-09-18T09:45:59.966Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c5861b94-204f-4964-a075-26c977a7fba3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:59.967Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"94b74d29-cfcc-40fb-98dc-a55f5836a974","rayCluster":"fail-bcffx"}
{"level":"error","ts":"2026-09-18T09:45:59.967Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"43bf8ea2-ae0b-4e8b-b532-210dcaae1e4a","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-bcffx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:59.967Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"43bf8ea2-ae0b-4e8b-b532-210dcaae1e4a"}
{"level":"error","ts":"2026-09-18T09:45:59.967Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"43bf8ea2-ae0b-4e8b-b532-210dcaae1e4a","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-bcffx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:45:59.967Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"94b74d29-cfcc-40fb-98dc-a55f5836a974"}
{"level":"info","ts":"2026-09-18T09:45:59.967Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"3e97ef98-685c-48e1-82ac-132f8b7c3331","namespacedName":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:45:59.967Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"3e97ef98-685c-48e1-82ac-132f8b7c3331","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:45:59.967Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"3e97ef98-685c-48e1-82ac-132f8b7c3331","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:00.066Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"312c6008-b27e-4a05-a32c-ebeab49015ab"}
{"level":"info","ts":"2026-09-18T09:46:00.066Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"312c6008-b27e-4a05-a32c-ebeab49015ab"}
{"level":"info","ts":"2026-09-18T09:46:00.066Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"312c6008-b27e-4a05-a32c-ebeab49015ab"}
{"level":"info","ts":"2026-09-18T09:46:00.066Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"312c6008-b27e-4a05-a32c-ebeab49015ab","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:00.066Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"312c6008-b27e-4a05-a32c-ebeab49015ab","error":"head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"error","ts":"2026-09-18T09:46:00.066Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"312c6008-b27e-4a05-a32c-ebeab49015ab","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:00.066Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"312c6008-b27e-4a05-a32c-ebeab49015ab","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"info","ts":"2026-09-18T09:46:00.066Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"312c6008-b27e-4a05-a32c-ebeab49015ab","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:00.066Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"312c6008-b27e-4a05-a32c-ebeab49015ab","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:00.066Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"312c6008-b27e-4a05-a32c-ebeab49015ab","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:00.066Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"312c6008-b27e-4a05-a32c-ebeab49015ab"}
{"level":"error","ts":"2026-09-18T09:46:00.066Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"312c6008-b27e-4a05-a32c-ebeab49015ab","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:00.066Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"541e0b26-089d-48b1-9081-20f3d1d66d92","name":"fail-bcffx-head"}
{"level":"info","ts":"2026-09-18T09:46:00.067Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6bd38411-4868-429e-b8b8-a00cc317d176","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:00.067Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6bd38411-4868-429e-b8b8-a00cc317d176","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:00.067Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6bd38411-4868-429e-b8b8-a00cc317d176","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:00.067Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"3e97ef98-685c-48e1-82ac-132f8b7c3331","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:00.067Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"a1df7d2e-1e90-42e4-8638-947d35ccfabb","namespacedName":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:00.067Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"a1df7d2e-1e90-42e4-8638-947d35ccfabb","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:00.067Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"a1df7d2e-1e90-42e4-8638-947d35ccfabb","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:00.067Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6bd38411-4868-429e-b8b8-a00cc317d176"}
{"level":"info","ts":"2026-09-18T09:46:00.067Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6bd38411-4868-429e-b8b8-a00cc317d176","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:00.067Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6bd38411-4868-429e-b8b8-a00cc317d176","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:00.067Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6bd38411-4868-429e-b8b8-a00cc317d176","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:00.068Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"96eda649-fef4-4674-93c7-f8b62aee7796","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:00.068Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"96eda649-fef4-4674-93c7-f8b62aee7796"}
{"level":"info","ts":"2026-09-18T09:46:00.078Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"a1df7d2e-1e90-42e4-8638-947d35ccfabb","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:00.083Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"541e0b26-089d-48b1-9081-20f3d1d66d92","name":"fail-bcffx-workers"}
{"level":"info","ts":"2026-09-18T09:46:00.083Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"541e0b26-089d-48b1-9081-20f3d1d66d92","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:00.083Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c0e239b5-aa7b-4e1c-8ce7-6e013ecae4a1","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:00.166Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"aec6a7e8-6a0b-4b71-b5ce-c710d9ee7cb0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:00.166Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"aec6a7e8-6a0b-4b71-b5ce-c710d9ee7cb0","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:00.167Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"aec6a7e8-6a0b-4b71-b5ce-c710d9ee7cb0","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:00.167Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"aec6a7e8-6a0b-4b71-b5ce-c710d9ee7cb0"}
{"level":"info","ts":"2026-09-18T09:46:00.167Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"aec6a7e8-6a0b-4b71-b5ce-c710d9ee7cb0","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:00.167Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"aec6a7e8-6a0b-4b71-b5ce-c710d9ee7cb0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:00.171Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c0e239b5-aa7b-4e1c-8ce7-6e013ecae4a1","name":"fail-bcffx-head"}
{"level":"info","ts":"2026-09-18T09:46:00.181Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c0e239b5-aa7b-4e1c-8ce7-6e013ecae4a1","name":"fail-bcffx-workers"}
{"level":"info","ts":"2026-09-18T09:46:00.181Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c0e239b5-aa7b-4e1c-8ce7-6e013ecae4a1","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:00.227Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d116f645-5952-4404-8a6c-af083d3f1d3c"}
{"level":"info","ts":"2026-09-18T09:46:00.227Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d116f645-5952-4404-8a6c-af083d3f1d3c"}
{"level":"info","ts":"2026-09-18T09:46:00.227Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d116f645-5952-4404-8a6c-af083d3f1d3c"}
{"level":"info","ts":"2026-09-18T09:46:00.227Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d116f645-5952-4404-8a6c-af083d3f1d3c","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:00.227Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d116f645-5952-4404-8a6c-af083d3f1d3c","error":"head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"error","ts":"2026-09-18T09:46:00.227Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d116f645-5952-4404-8a6c-af083d3f1d3c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:00.227Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d116f645-5952-4404-8a6c-af083d3f1d3c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"info","ts":"2026-09-18T09:46:00.227Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d116f645-5952-4404-8a6c-af083d3f1d3c","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:00.227Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d116f645-5952-4404-8a6c-af083d3f1d3c","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:00.227Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d116f645-5952-4404-8a6c-af083d3f1d3c","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:00.227Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d116f645-5952-4404-8a6c-af083d3f1d3c"}
{"level":"error","ts":"2026-09-18T09:46:00.227Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d116f645-5952-4404-8a6c-af083d3f1d3c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:00.547Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3e219850-a3c6-412d-be0b-f1c0158f5280"}
{"level":"info","ts":"2026-09-18T09:46:00.547Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3e219850-a3c6-412d-be0b-f1c0158f5280"}
{"level":"info","ts":"2026-09-18T09:46:00.547Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3e219850-a3c6-412d-be0b-f1c0158f5280"}
{"level":"info","ts":"2026-09-18T09:46:00.547Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3e219850-a3c6-412d-be0b-f1c0158f5280","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:00.547Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3e219850-a3c6-412d-be0b-f1c0158f5280","error":"head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"error","ts":"2026-09-18T09:46:00.547Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3e219850-a3c6-412d-be0b-f1c0158f5280","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:00.547Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3e219850-a3c6-412d-be0b-f1c0158f5280","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"info","ts":"2026-09-18T09:46:00.547Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3e219850-a3c6-412d-be0b-f1c0158f5280","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:00.547Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3e219850-a3c6-412d-be0b-f1c0158f5280","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:00.547Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3e219850-a3c6-412d-be0b-f1c0158f5280","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:00.548Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3e219850-a3c6-412d-be0b-f1c0158f5280"}
{"level":"error","ts":"2026-09-18T09:46:00.548Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3e219850-a3c6-412d-be0b-f1c0158f5280","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:01.188Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0e7139d2-0abd-4b54-9882-159b81f6cf20"}
{"level":"info","ts":"2026-09-18T09:46:01.188Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0e7139d2-0abd-4b54-9882-159b81f6cf20"}
{"level":"info","ts":"2026-09-18T09:46:01.188Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0e7139d2-0abd-4b54-9882-159b81f6cf20"}
{"level":"info","ts":"2026-09-18T09:46:01.188Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0e7139d2-0abd-4b54-9882-159b81f6cf20","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:01.189Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0e7139d2-0abd-4b54-9882-159b81f6cf20","error":"head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"error","ts":"2026-09-18T09:46:01.189Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0e7139d2-0abd-4b54-9882-159b81f6cf20","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:01.189Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0e7139d2-0abd-4b54-9882-159b81f6cf20","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found"}
{"level":"info","ts":"2026-09-18T09:46:01.189Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0e7139d2-0abd-4b54-9882-159b81f6cf20","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:01.189Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0e7139d2-0abd-4b54-9882-159b81f6cf20","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:01.189Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0e7139d2-0abd-4b54-9882-159b81f6cf20","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:01.189Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0e7139d2-0abd-4b54-9882-159b81f6cf20"}
{"level":"error","ts":"2026-09-18T09:46:01.189Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0e7139d2-0abd-4b54-9882-159b81f6cf20","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-bcffx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:02.390Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"ec8975c2-1b77-4e55-a697-cb8d0a4cec32","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:02.390Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"ec8975c2-1b77-4e55-a697-cb8d0a4cec32","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:45:59 +0000 UTC","Now":"2026-09-18T09:46:02.390Z","ShutdownTime":"2026-09-18T09:45:59.000Z"}
{"level":"info","ts":"2026-09-18T09:46:02.390Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"ec8975c2-1b77-4e55-a697-cb8d0a4cec32","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:02.390Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"ec8975c2-1b77-4e55-a697-cb8d0a4cec32","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T09:46:02.390Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"ec8975c2-1b77-4e55-a697-cb8d0a4cec32","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:46:02.469Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9"}
{"level":"info","ts":"2026-09-18T09:46:02.469Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9"}
{"level":"info","ts":"2026-09-18T09:46:02.469Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9"}
{"level":"info","ts":"2026-09-18T09:46:02.469Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:02.469Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9"}
{"level":"info","ts":"2026-09-18T09:46:02.469Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:46:02.469Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9"}
{"level":"info","ts":"2026-09-18T09:46:02.469Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:02.469Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","cluster":"fail-bcffx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:46:02.469Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","cluster":"fail-bcffx","authType":"IntegratedOAuth","serviceAccount":"fail-bcffx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:46:02.469Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-bcffx","ray.io/group":"headgroup","ray.io/identifier":"fail-bcffx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:46:02.469Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:46:02.470Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:46:02.470Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:46:02.470Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","enabled":true}
{"level":"info","ts":"2026-09-18T09:46:02.510Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","name":"fail-bcffx-head-s6qj5"}
{"level":"info","ts":"2026-09-18T09:46:02.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:02.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:02.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:46:02.510Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:46:02.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:02.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:46:02.510Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9"}
{"level":"info","ts":"2026-09-18T09:46:02.510Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","nodeType":"worker","rayStartParams":{"address":"fail-bcffx-head-svc.test-ns-q2lj2.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:46:02.510Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","rayStartCmd":"ray start  --address=fail-bcffx-head-svc.test-ns-q2lj2.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:46:02.510Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-bcffx-head-svc.test-ns-q2lj2.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:46:02.510Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","enabled":true}
{"level":"info","ts":"2026-09-18T09:46:02.571Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6bd2d48f-ea72-4b25-9191-49f2800c3a8d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:02.571Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6bd2d48f-ea72-4b25-9191-49f2800c3a8d","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:02.571Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6bd2d48f-ea72-4b25-9191-49f2800c3a8d","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:02.571Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6bd2d48f-ea72-4b25-9191-49f2800c3a8d"}
{"level":"info","ts":"2026-09-18T09:46:02.571Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6bd2d48f-ea72-4b25-9191-49f2800c3a8d","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:02.571Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6bd2d48f-ea72-4b25-9191-49f2800c3a8d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:02.583Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","name":"fail-bcffx-small-group-worker-dcw8j"}
{"level":"info","ts":"2026-09-18T09:46:02.583Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:02.583Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"adba02fb-df59-469f-9e28-a58a6003b1f9","name":"fail-bcffx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:02.601Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"b72636d5-b5f6-4acd-a440-46c7bb717590","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:02.601Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"646fb373-9f35-464a-9d16-7c529828496f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:02.601Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"646fb373-9f35-464a-9d16-7c529828496f","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:02.601Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"646fb373-9f35-464a-9d16-7c529828496f","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:02.601Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"b72636d5-b5f6-4acd-a440-46c7bb717590","rayCluster":"fail-bcffx","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:46:02.601Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"646fb373-9f35-464a-9d16-7c529828496f"}
{"level":"info","ts":"2026-09-18T09:46:02.601Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"646fb373-9f35-464a-9d16-7c529828496f","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:02.601Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"058027e9-8f9f-4ce5-84e8-cd4d11146de0","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:02.601Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"646fb373-9f35-464a-9d16-7c529828496f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:45:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:02.601Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"646fb373-9f35-464a-9d16-7c529828496f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3f8b4209-2de7-4cd9-afee-8c60143c6a96"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3f8b4209-2de7-4cd9-afee-8c60143c6a96"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3f8b4209-2de7-4cd9-afee-8c60143c6a96"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3f8b4209-2de7-4cd9-afee-8c60143c6a96","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3f8b4209-2de7-4cd9-afee-8c60143c6a96","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3f8b4209-2de7-4cd9-afee-8c60143c6a96","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3f8b4209-2de7-4cd9-afee-8c60143c6a96","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3f8b4209-2de7-4cd9-afee-8c60143c6a96","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3f8b4209-2de7-4cd9-afee-8c60143c6a96","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3f8b4209-2de7-4cd9-afee-8c60143c6a96","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3f8b4209-2de7-4cd9-afee-8c60143c6a96","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3f8b4209-2de7-4cd9-afee-8c60143c6a96","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3f8b4209-2de7-4cd9-afee-8c60143c6a96","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3f8b4209-2de7-4cd9-afee-8c60143c6a96","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3f8b4209-2de7-4cd9-afee-8c60143c6a96","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3f8b4209-2de7-4cd9-afee-8c60143c6a96","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f6257ed4-7d09-4990-9d6d-2d8d914e018a"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f6257ed4-7d09-4990-9d6d-2d8d914e018a"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f6257ed4-7d09-4990-9d6d-2d8d914e018a"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f6257ed4-7d09-4990-9d6d-2d8d914e018a","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f6257ed4-7d09-4990-9d6d-2d8d914e018a","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f6257ed4-7d09-4990-9d6d-2d8d914e018a","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f6257ed4-7d09-4990-9d6d-2d8d914e018a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f6257ed4-7d09-4990-9d6d-2d8d914e018a","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f6257ed4-7d09-4990-9d6d-2d8d914e018a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f6257ed4-7d09-4990-9d6d-2d8d914e018a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f6257ed4-7d09-4990-9d6d-2d8d914e018a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f6257ed4-7d09-4990-9d6d-2d8d914e018a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f6257ed4-7d09-4990-9d6d-2d8d914e018a","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:02.602Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f6257ed4-7d09-4990-9d6d-2d8d914e018a","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:02.603Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f6257ed4-7d09-4990-9d6d-2d8d914e018a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:02.603Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f6257ed4-7d09-4990-9d6d-2d8d914e018a","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:02.617Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"5d9a09c7-13ca-4076-aae2-0bbb99086737","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:02.617Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"5d9a09c7-13ca-4076-aae2-0bbb99086737","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:02.617Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"5d9a09c7-13ca-4076-aae2-0bbb99086737","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:02.617Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"5d9a09c7-13ca-4076-aae2-0bbb99086737"}
{"level":"info","ts":"2026-09-18T09:46:02.617Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"5d9a09c7-13ca-4076-aae2-0bbb99086737","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:02.617Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"5d9a09c7-13ca-4076-aae2-0bbb99086737","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:02.622Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"30cdd827-f7d7-4dba-ab4d-e8687841f6d3"}
{"level":"info","ts":"2026-09-18T09:46:02.622Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"30cdd827-f7d7-4dba-ab4d-e8687841f6d3"}
{"level":"info","ts":"2026-09-18T09:46:02.622Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"30cdd827-f7d7-4dba-ab4d-e8687841f6d3"}
{"level":"info","ts":"2026-09-18T09:46:02.622Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"30cdd827-f7d7-4dba-ab4d-e8687841f6d3","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:02.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"30cdd827-f7d7-4dba-ab4d-e8687841f6d3","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:02.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"30cdd827-f7d7-4dba-ab4d-e8687841f6d3","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:02.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"30cdd827-f7d7-4dba-ab4d-e8687841f6d3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:02.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"30cdd827-f7d7-4dba-ab4d-e8687841f6d3","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:02.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"30cdd827-f7d7-4dba-ab4d-e8687841f6d3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:02.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"30cdd827-f7d7-4dba-ab4d-e8687841f6d3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:02.622Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"30cdd827-f7d7-4dba-ab4d-e8687841f6d3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:02.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"30cdd827-f7d7-4dba-ab4d-e8687841f6d3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:02.622Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"30cdd827-f7d7-4dba-ab4d-e8687841f6d3","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:02.622Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"30cdd827-f7d7-4dba-ab4d-e8687841f6d3","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:02.622Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"30cdd827-f7d7-4dba-ab4d-e8687841f6d3","name":"fail-bcffx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:02.628Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"058027e9-8f9f-4ce5-84e8-cd4d11146de0","name":"fail-bcffx-head"}
{"level":"info","ts":"2026-09-18T09:46:02.646Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"058027e9-8f9f-4ce5-84e8-cd4d11146de0","name":"fail-bcffx-workers"}
{"level":"info","ts":"2026-09-18T09:46:02.646Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"058027e9-8f9f-4ce5-84e8-cd4d11146de0","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:02.647Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"34aeaca6-2062-4441-a9a0-e743284e79fd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:02.647Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"34aeaca6-2062-4441-a9a0-e743284e79fd","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:02.647Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"34aeaca6-2062-4441-a9a0-e743284e79fd","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:02.647Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"34aeaca6-2062-4441-a9a0-e743284e79fd"}
{"level":"info","ts":"2026-09-18T09:46:02.647Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"34aeaca6-2062-4441-a9a0-e743284e79fd","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:02.647Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"34aeaca6-2062-4441-a9a0-e743284e79fd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:02.647Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"34aeaca6-2062-4441-a9a0-e743284e79fd","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:02.648Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"1d76b1dc-8e8e-4116-bf94-624715a048f2","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:02.648Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"1d76b1dc-8e8e-4116-bf94-624715a048f2","rayCluster":"fail-bcffx","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:46:02.648Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"1dc741dc-0de5-483a-b47c-5aa305328990","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"1ce8d8d3-a683-48b2-8b3f-a3fd9b95d0a9"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"1ce8d8d3-a683-48b2-8b3f-a3fd9b95d0a9"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"1ce8d8d3-a683-48b2-8b3f-a3fd9b95d0a9"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"1ce8d8d3-a683-48b2-8b3f-a3fd9b95d0a9","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"1ce8d8d3-a683-48b2-8b3f-a3fd9b95d0a9","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"1ce8d8d3-a683-48b2-8b3f-a3fd9b95d0a9","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"1ce8d8d3-a683-48b2-8b3f-a3fd9b95d0a9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"1ce8d8d3-a683-48b2-8b3f-a3fd9b95d0a9","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"1ce8d8d3-a683-48b2-8b3f-a3fd9b95d0a9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"1ce8d8d3-a683-48b2-8b3f-a3fd9b95d0a9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"1ce8d8d3-a683-48b2-8b3f-a3fd9b95d0a9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"1ce8d8d3-a683-48b2-8b3f-a3fd9b95d0a9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"1ce8d8d3-a683-48b2-8b3f-a3fd9b95d0a9","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"1ce8d8d3-a683-48b2-8b3f-a3fd9b95d0a9","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"1ce8d8d3-a683-48b2-8b3f-a3fd9b95d0a9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"1ce8d8d3-a683-48b2-8b3f-a3fd9b95d0a9","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8a5bd88e-a00c-4e94-8408-ab033960aa0c"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8a5bd88e-a00c-4e94-8408-ab033960aa0c"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8a5bd88e-a00c-4e94-8408-ab033960aa0c"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8a5bd88e-a00c-4e94-8408-ab033960aa0c","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8a5bd88e-a00c-4e94-8408-ab033960aa0c","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8a5bd88e-a00c-4e94-8408-ab033960aa0c","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8a5bd88e-a00c-4e94-8408-ab033960aa0c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8a5bd88e-a00c-4e94-8408-ab033960aa0c","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8a5bd88e-a00c-4e94-8408-ab033960aa0c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8a5bd88e-a00c-4e94-8408-ab033960aa0c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8a5bd88e-a00c-4e94-8408-ab033960aa0c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8a5bd88e-a00c-4e94-8408-ab033960aa0c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8a5bd88e-a00c-4e94-8408-ab033960aa0c","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:02.666Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8a5bd88e-a00c-4e94-8408-ab033960aa0c","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:02.667Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8a5bd88e-a00c-4e94-8408-ab033960aa0c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:02.667Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8a5bd88e-a00c-4e94-8408-ab033960aa0c","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:02.667Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"5f72c4fd-bb2c-4189-94d0-c7578c613026","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:02.667Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"5f72c4fd-bb2c-4189-94d0-c7578c613026","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:02.667Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"5f72c4fd-bb2c-4189-94d0-c7578c613026","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:02.667Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"5f72c4fd-bb2c-4189-94d0-c7578c613026"}
{"level":"info","ts":"2026-09-18T09:46:02.667Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"5f72c4fd-bb2c-4189-94d0-c7578c613026","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:02.667Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"5f72c4fd-bb2c-4189-94d0-c7578c613026","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:02.678Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"1dc741dc-0de5-483a-b47c-5aa305328990","name":"fail-bcffx-head"}
{"level":"info","ts":"2026-09-18T09:46:02.694Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"1dc741dc-0de5-483a-b47c-5aa305328990","name":"fail-bcffx-workers"}
{"level":"info","ts":"2026-09-18T09:46:02.694Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"1dc741dc-0de5-483a-b47c-5aa305328990","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:03.117Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"250fa323-aa22-48ad-8f3c-dfad2e51b47c"}
{"level":"info","ts":"2026-09-18T09:46:03.117Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"250fa323-aa22-48ad-8f3c-dfad2e51b47c"}
{"level":"info","ts":"2026-09-18T09:46:03.117Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"250fa323-aa22-48ad-8f3c-dfad2e51b47c"}
{"level":"info","ts":"2026-09-18T09:46:03.117Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"250fa323-aa22-48ad-8f3c-dfad2e51b47c","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:03.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"250fa323-aa22-48ad-8f3c-dfad2e51b47c","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:03.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"250fa323-aa22-48ad-8f3c-dfad2e51b47c","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:03.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"250fa323-aa22-48ad-8f3c-dfad2e51b47c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:03.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"250fa323-aa22-48ad-8f3c-dfad2e51b47c","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:03.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"250fa323-aa22-48ad-8f3c-dfad2e51b47c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:03.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"250fa323-aa22-48ad-8f3c-dfad2e51b47c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:03.117Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"250fa323-aa22-48ad-8f3c-dfad2e51b47c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:03.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"250fa323-aa22-48ad-8f3c-dfad2e51b47c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:03.118Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"250fa323-aa22-48ad-8f3c-dfad2e51b47c","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:03.118Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"250fa323-aa22-48ad-8f3c-dfad2e51b47c","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:03.118Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"250fa323-aa22-48ad-8f3c-dfad2e51b47c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:03.118Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"250fa323-aa22-48ad-8f3c-dfad2e51b47c","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:03.191Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c17ae26e-70ae-4146-9144-10716f1eb4f0"}
{"level":"info","ts":"2026-09-18T09:46:03.191Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c17ae26e-70ae-4146-9144-10716f1eb4f0"}
{"level":"info","ts":"2026-09-18T09:46:03.191Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c17ae26e-70ae-4146-9144-10716f1eb4f0"}
{"level":"info","ts":"2026-09-18T09:46:03.191Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c17ae26e-70ae-4146-9144-10716f1eb4f0","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:03.191Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c17ae26e-70ae-4146-9144-10716f1eb4f0","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:03.191Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c17ae26e-70ae-4146-9144-10716f1eb4f0","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:03.191Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c17ae26e-70ae-4146-9144-10716f1eb4f0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:03.191Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c17ae26e-70ae-4146-9144-10716f1eb4f0","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:03.191Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c17ae26e-70ae-4146-9144-10716f1eb4f0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:03.191Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c17ae26e-70ae-4146-9144-10716f1eb4f0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:03.191Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c17ae26e-70ae-4146-9144-10716f1eb4f0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:03.191Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c17ae26e-70ae-4146-9144-10716f1eb4f0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:03.191Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c17ae26e-70ae-4146-9144-10716f1eb4f0","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:03.192Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c17ae26e-70ae-4146-9144-10716f1eb4f0","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:03.192Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c17ae26e-70ae-4146-9144-10716f1eb4f0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:03.192Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"c17ae26e-70ae-4146-9144-10716f1eb4f0","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:03.319Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f01e2a0f-1d66-4c4b-b416-38769df3bdfe"}
{"level":"info","ts":"2026-09-18T09:46:03.319Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f01e2a0f-1d66-4c4b-b416-38769df3bdfe"}
{"level":"info","ts":"2026-09-18T09:46:03.319Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f01e2a0f-1d66-4c4b-b416-38769df3bdfe"}
{"level":"info","ts":"2026-09-18T09:46:03.319Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f01e2a0f-1d66-4c4b-b416-38769df3bdfe","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:03.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f01e2a0f-1d66-4c4b-b416-38769df3bdfe","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:03.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f01e2a0f-1d66-4c4b-b416-38769df3bdfe","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:03.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f01e2a0f-1d66-4c4b-b416-38769df3bdfe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:03.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f01e2a0f-1d66-4c4b-b416-38769df3bdfe","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:03.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f01e2a0f-1d66-4c4b-b416-38769df3bdfe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:03.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f01e2a0f-1d66-4c4b-b416-38769df3bdfe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:03.319Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f01e2a0f-1d66-4c4b-b416-38769df3bdfe","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:03.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f01e2a0f-1d66-4c4b-b416-38769df3bdfe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:03.319Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f01e2a0f-1d66-4c4b-b416-38769df3bdfe","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:03.319Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f01e2a0f-1d66-4c4b-b416-38769df3bdfe","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:03.319Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f01e2a0f-1d66-4c4b-b416-38769df3bdfe","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:03.319Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f01e2a0f-1d66-4c4b-b416-38769df3bdfe","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:04.149Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9995173d-30c4-40cc-8124-e5dfebf36949"}
{"level":"info","ts":"2026-09-18T09:46:04.149Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9995173d-30c4-40cc-8124-e5dfebf36949"}
{"level":"info","ts":"2026-09-18T09:46:04.149Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9995173d-30c4-40cc-8124-e5dfebf36949"}
{"level":"info","ts":"2026-09-18T09:46:04.149Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9995173d-30c4-40cc-8124-e5dfebf36949","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:04.149Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9995173d-30c4-40cc-8124-e5dfebf36949","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:04.149Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9995173d-30c4-40cc-8124-e5dfebf36949","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:04.149Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9995173d-30c4-40cc-8124-e5dfebf36949","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:04.149Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9995173d-30c4-40cc-8124-e5dfebf36949","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:04.149Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9995173d-30c4-40cc-8124-e5dfebf36949","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:04.149Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9995173d-30c4-40cc-8124-e5dfebf36949","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:04.150Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9995173d-30c4-40cc-8124-e5dfebf36949","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:04.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9995173d-30c4-40cc-8124-e5dfebf36949","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:04.150Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9995173d-30c4-40cc-8124-e5dfebf36949","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:04.150Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9995173d-30c4-40cc-8124-e5dfebf36949","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:04.150Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"9995173d-30c4-40cc-8124-e5dfebf36949","name":"fail-bcffx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:04.163Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"f20346f0-4eca-4aa7-bbce-355f2341dfaf","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:04.163Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"0d77404d-07dc-4260-8a3d-bcdbe69727b5","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:04.163Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"0d77404d-07dc-4260-8a3d-bcdbe69727b5","pod":"fail-bcffx-head-s6qj5","ip":"10.133.0.29"}
{"level":"info","ts":"2026-09-18T09:46:04.163Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"0d77404d-07dc-4260-8a3d-bcdbe69727b5","pod":"fail-bcffx-small-group-worker-dcw8j","ip":"10.132.0.31"}
{"level":"info","ts":"2026-09-18T09:46:04.164Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"b5a4389b-2815-46ab-8575-b6e8cc2cbb5f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:04.164Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"b5a4389b-2815-46ab-8575-b6e8cc2cbb5f","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:04.164Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"b5a4389b-2815-46ab-8575-b6e8cc2cbb5f","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:04.164Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"b5a4389b-2815-46ab-8575-b6e8cc2cbb5f"}
{"level":"info","ts":"2026-09-18T09:46:04.164Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"b5a4389b-2815-46ab-8575-b6e8cc2cbb5f","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:04.164Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"b5a4389b-2815-46ab-8575-b6e8cc2cbb5f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:04.164Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"b5a4389b-2815-46ab-8575-b6e8cc2cbb5f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:04.179Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f2960102-393d-4b28-8d06-007eb417d38c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:04.179Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f2960102-393d-4b28-8d06-007eb417d38c","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:04.179Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f2960102-393d-4b28-8d06-007eb417d38c","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:04.179Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f2960102-393d-4b28-8d06-007eb417d38c"}
{"level":"info","ts":"2026-09-18T09:46:04.179Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f2960102-393d-4b28-8d06-007eb417d38c","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:04.179Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f2960102-393d-4b28-8d06-007eb417d38c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:04.184Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"f20346f0-4eca-4aa7-bbce-355f2341dfaf","name":"fail-bcffx-head"}
{"level":"info","ts":"2026-09-18T09:46:04.185Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"0d77404d-07dc-4260-8a3d-bcdbe69727b5","certificate":"ray-head-cert-fail-bcffx","dnsNames":["fail-bcffx-head-svc","localhost","fail-bcffx-head-svc.test-ns-q2lj2.svc","fail-bcffx-head-svc.test-ns-q2lj2.svc.cluster.local"],"ipAddresses":["10.133.0.29","10.132.0.31","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:46:04.201Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"f20346f0-4eca-4aa7-bbce-355f2341dfaf","name":"fail-bcffx-workers"}
{"level":"info","ts":"2026-09-18T09:46:04.201Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"f20346f0-4eca-4aa7-bbce-355f2341dfaf","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:04.205Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"0d77404d-07dc-4260-8a3d-bcdbe69727b5","certificate":"ray-worker-cert-fail-bcffx","dnsNames":["fail-bcffx-worker-svc","localhost","fail-bcffx-small-group","fail-bcffx-worker-svc.test-ns-q2lj2.svc","fail-bcffx-worker-svc.test-ns-q2lj2.svc.cluster.local","*.fail-bcffx-worker-svc.test-ns-q2lj2.svc","*.fail-bcffx-worker-svc.test-ns-q2lj2.svc.cluster.local","*-worker-*.test-ns-q2lj2.svc","*-worker-*.test-ns-q2lj2.svc.cluster.local"],"ipAddresses":["10.133.0.29","10.132.0.31","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:46:04.205Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"0d77404d-07dc-4260-8a3d-bcdbe69727b5"}
{"level":"info","ts":"2026-09-18T09:46:04.603Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed4dbff8-22b4-44c9-8ea0-582115d7100f"}
{"level":"info","ts":"2026-09-18T09:46:04.603Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed4dbff8-22b4-44c9-8ea0-582115d7100f"}
{"level":"info","ts":"2026-09-18T09:46:04.603Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed4dbff8-22b4-44c9-8ea0-582115d7100f"}
{"level":"info","ts":"2026-09-18T09:46:04.603Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed4dbff8-22b4-44c9-8ea0-582115d7100f","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:04.603Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed4dbff8-22b4-44c9-8ea0-582115d7100f","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:04.603Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed4dbff8-22b4-44c9-8ea0-582115d7100f","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:04.603Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed4dbff8-22b4-44c9-8ea0-582115d7100f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:04.603Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed4dbff8-22b4-44c9-8ea0-582115d7100f","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:04.603Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed4dbff8-22b4-44c9-8ea0-582115d7100f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:04.603Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed4dbff8-22b4-44c9-8ea0-582115d7100f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:04.603Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed4dbff8-22b4-44c9-8ea0-582115d7100f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:04.603Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed4dbff8-22b4-44c9-8ea0-582115d7100f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:04.603Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed4dbff8-22b4-44c9-8ea0-582115d7100f","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:04.603Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed4dbff8-22b4-44c9-8ea0-582115d7100f","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:04.603Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed4dbff8-22b4-44c9-8ea0-582115d7100f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:04.603Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed4dbff8-22b4-44c9-8ea0-582115d7100f","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:05.572Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"da835cbc-e901-424b-8737-77dfa26b19a5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:05.572Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"da835cbc-e901-424b-8737-77dfa26b19a5","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:05.572Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"da835cbc-e901-424b-8737-77dfa26b19a5","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:05.572Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"da835cbc-e901-424b-8737-77dfa26b19a5"}
{"level":"info","ts":"2026-09-18T09:46:05.572Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"da835cbc-e901-424b-8737-77dfa26b19a5","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:05.572Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"da835cbc-e901-424b-8737-77dfa26b19a5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:08.340Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"64a7ed40-826a-40d8-a5dd-a2afc70ba645"}
{"level":"info","ts":"2026-09-18T09:46:08.340Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"64a7ed40-826a-40d8-a5dd-a2afc70ba645"}
{"level":"info","ts":"2026-09-18T09:46:08.340Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"64a7ed40-826a-40d8-a5dd-a2afc70ba645"}
{"level":"info","ts":"2026-09-18T09:46:08.340Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"64a7ed40-826a-40d8-a5dd-a2afc70ba645","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:08.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"64a7ed40-826a-40d8-a5dd-a2afc70ba645","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:08.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"64a7ed40-826a-40d8-a5dd-a2afc70ba645","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:08.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"64a7ed40-826a-40d8-a5dd-a2afc70ba645","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:08.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"64a7ed40-826a-40d8-a5dd-a2afc70ba645","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:08.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"64a7ed40-826a-40d8-a5dd-a2afc70ba645","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:08.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"64a7ed40-826a-40d8-a5dd-a2afc70ba645","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:08.340Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"64a7ed40-826a-40d8-a5dd-a2afc70ba645","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:08.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"64a7ed40-826a-40d8-a5dd-a2afc70ba645","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:08.340Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"64a7ed40-826a-40d8-a5dd-a2afc70ba645","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:08.340Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"64a7ed40-826a-40d8-a5dd-a2afc70ba645","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:08.340Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"64a7ed40-826a-40d8-a5dd-a2afc70ba645","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:08.340Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"64a7ed40-826a-40d8-a5dd-a2afc70ba645","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:08.573Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"916f93d3-202c-43ed-b053-c380851ac633","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:08.573Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"916f93d3-202c-43ed-b053-c380851ac633","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:08.573Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"916f93d3-202c-43ed-b053-c380851ac633","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:08.573Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"916f93d3-202c-43ed-b053-c380851ac633"}
{"level":"info","ts":"2026-09-18T09:46:08.573Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"916f93d3-202c-43ed-b053-c380851ac633","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:08.573Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"916f93d3-202c-43ed-b053-c380851ac633","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:09.170Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8e776693-5242-481c-98fa-8206e45f70d4"}
{"level":"info","ts":"2026-09-18T09:46:09.170Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8e776693-5242-481c-98fa-8206e45f70d4"}
{"level":"info","ts":"2026-09-18T09:46:09.170Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8e776693-5242-481c-98fa-8206e45f70d4"}
{"level":"info","ts":"2026-09-18T09:46:09.170Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8e776693-5242-481c-98fa-8206e45f70d4","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:09.170Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8e776693-5242-481c-98fa-8206e45f70d4","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:09.170Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8e776693-5242-481c-98fa-8206e45f70d4","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:09.170Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8e776693-5242-481c-98fa-8206e45f70d4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:09.170Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8e776693-5242-481c-98fa-8206e45f70d4","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:09.170Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8e776693-5242-481c-98fa-8206e45f70d4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:09.170Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8e776693-5242-481c-98fa-8206e45f70d4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:09.170Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8e776693-5242-481c-98fa-8206e45f70d4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:09.170Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8e776693-5242-481c-98fa-8206e45f70d4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:09.170Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8e776693-5242-481c-98fa-8206e45f70d4","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:09.170Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8e776693-5242-481c-98fa-8206e45f70d4","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:09.170Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8e776693-5242-481c-98fa-8206e45f70d4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:09.170Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8e776693-5242-481c-98fa-8206e45f70d4","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:09.350Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"60aa55a3-ead0-4841-86cc-b5aa5dda1441"}
{"level":"info","ts":"2026-09-18T09:46:09.350Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"60aa55a3-ead0-4841-86cc-b5aa5dda1441"}
{"level":"info","ts":"2026-09-18T09:46:09.350Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"60aa55a3-ead0-4841-86cc-b5aa5dda1441"}
{"level":"info","ts":"2026-09-18T09:46:09.350Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"60aa55a3-ead0-4841-86cc-b5aa5dda1441","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:09.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"60aa55a3-ead0-4841-86cc-b5aa5dda1441","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:09.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"60aa55a3-ead0-4841-86cc-b5aa5dda1441","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:09.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"60aa55a3-ead0-4841-86cc-b5aa5dda1441","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:09.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"60aa55a3-ead0-4841-86cc-b5aa5dda1441","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:09.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"60aa55a3-ead0-4841-86cc-b5aa5dda1441","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:09.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"60aa55a3-ead0-4841-86cc-b5aa5dda1441","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:09.350Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"60aa55a3-ead0-4841-86cc-b5aa5dda1441","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:09.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"60aa55a3-ead0-4841-86cc-b5aa5dda1441","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:09.350Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"60aa55a3-ead0-4841-86cc-b5aa5dda1441","podName":"fail-bcffx-small-group-worker-dcw8j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:09.350Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"60aa55a3-ead0-4841-86cc-b5aa5dda1441","podName":"fail-bcffx-small-group-worker-dcw8j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:09.350Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"60aa55a3-ead0-4841-86cc-b5aa5dda1441","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:09.350Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"60aa55a3-ead0-4841-86cc-b5aa5dda1441","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:10.176Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8d1bfb82-7f61-4e40-a77b-166e16ccd58a"}
{"level":"info","ts":"2026-09-18T09:46:10.176Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8d1bfb82-7f61-4e40-a77b-166e16ccd58a"}
{"level":"info","ts":"2026-09-18T09:46:10.176Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8d1bfb82-7f61-4e40-a77b-166e16ccd58a"}
{"level":"info","ts":"2026-09-18T09:46:10.176Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8d1bfb82-7f61-4e40-a77b-166e16ccd58a","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:10.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8d1bfb82-7f61-4e40-a77b-166e16ccd58a","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:10.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8d1bfb82-7f61-4e40-a77b-166e16ccd58a","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:10.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8d1bfb82-7f61-4e40-a77b-166e16ccd58a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:10.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8d1bfb82-7f61-4e40-a77b-166e16ccd58a","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:10.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8d1bfb82-7f61-4e40-a77b-166e16ccd58a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:10.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8d1bfb82-7f61-4e40-a77b-166e16ccd58a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:10.176Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8d1bfb82-7f61-4e40-a77b-166e16ccd58a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:10.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8d1bfb82-7f61-4e40-a77b-166e16ccd58a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:10.176Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8d1bfb82-7f61-4e40-a77b-166e16ccd58a","podName":"fail-bcffx-small-group-worker-dcw8j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:10.176Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8d1bfb82-7f61-4e40-a77b-166e16ccd58a","podName":"fail-bcffx-small-group-worker-dcw8j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:10.177Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8d1bfb82-7f61-4e40-a77b-166e16ccd58a","name":"fail-bcffx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:10.190Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"3c0722c1-8f83-4904-9944-c8c74d405743","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:10.190Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"2b790030-0558-4506-9e60-3bb2b1da7306","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:10.190Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"3c0722c1-8f83-4904-9944-c8c74d405743","pod":"fail-bcffx-head-s6qj5","ip":"10.133.0.29"}
{"level":"info","ts":"2026-09-18T09:46:10.190Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"3c0722c1-8f83-4904-9944-c8c74d405743","pod":"fail-bcffx-small-group-worker-dcw8j","ip":"10.132.0.31"}
{"level":"info","ts":"2026-09-18T09:46:10.190Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"3c0722c1-8f83-4904-9944-c8c74d405743","rayCluster":"fail-bcffx","podIPs":["10.133.0.29","10.132.0.31"]}
{"level":"info","ts":"2026-09-18T09:46:10.190Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"73fb4ac2-8f3e-44ba-b84d-977d4da9f1e3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:10.190Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"73fb4ac2-8f3e-44ba-b84d-977d4da9f1e3","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:10.190Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"73fb4ac2-8f3e-44ba-b84d-977d4da9f1e3","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:10.190Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"73fb4ac2-8f3e-44ba-b84d-977d4da9f1e3"}
{"level":"info","ts":"2026-09-18T09:46:10.190Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"73fb4ac2-8f3e-44ba-b84d-977d4da9f1e3","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:10.190Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"73fb4ac2-8f3e-44ba-b84d-977d4da9f1e3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:10.191Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"73fb4ac2-8f3e-44ba-b84d-977d4da9f1e3","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:10.204Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"2b790030-0558-4506-9e60-3bb2b1da7306","name":"fail-bcffx-head"}
{"level":"info","ts":"2026-09-18T09:46:10.206Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f1580ff3-572b-4859-87fe-37607198b5d7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:10.206Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f1580ff3-572b-4859-87fe-37607198b5d7","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:10.206Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f1580ff3-572b-4859-87fe-37607198b5d7","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:10.206Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f1580ff3-572b-4859-87fe-37607198b5d7"}
{"level":"info","ts":"2026-09-18T09:46:10.206Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f1580ff3-572b-4859-87fe-37607198b5d7","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:10.206Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f1580ff3-572b-4859-87fe-37607198b5d7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:10.216Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"2b790030-0558-4506-9e60-3bb2b1da7306","name":"fail-bcffx-workers"}
{"level":"info","ts":"2026-09-18T09:46:10.216Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"2b790030-0558-4506-9e60-3bb2b1da7306","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:11.574Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"04795e9f-b347-46ea-9aa4-0f0966f6fed3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:11.574Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"04795e9f-b347-46ea-9aa4-0f0966f6fed3","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:11.574Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"04795e9f-b347-46ea-9aa4-0f0966f6fed3","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:11.575Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"04795e9f-b347-46ea-9aa4-0f0966f6fed3"}
{"level":"info","ts":"2026-09-18T09:46:11.575Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"04795e9f-b347-46ea-9aa4-0f0966f6fed3","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:11.575Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"04795e9f-b347-46ea-9aa4-0f0966f6fed3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:12.191Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed1277a0-073d-4eb8-8cf2-3634943ac3e0"}
{"level":"info","ts":"2026-09-18T09:46:12.191Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed1277a0-073d-4eb8-8cf2-3634943ac3e0"}
{"level":"info","ts":"2026-09-18T09:46:12.191Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed1277a0-073d-4eb8-8cf2-3634943ac3e0"}
{"level":"info","ts":"2026-09-18T09:46:12.191Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed1277a0-073d-4eb8-8cf2-3634943ac3e0","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:12.191Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed1277a0-073d-4eb8-8cf2-3634943ac3e0","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:12.191Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed1277a0-073d-4eb8-8cf2-3634943ac3e0","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:12.191Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed1277a0-073d-4eb8-8cf2-3634943ac3e0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:12.191Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed1277a0-073d-4eb8-8cf2-3634943ac3e0","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:12.191Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed1277a0-073d-4eb8-8cf2-3634943ac3e0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:12.191Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed1277a0-073d-4eb8-8cf2-3634943ac3e0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:12.191Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed1277a0-073d-4eb8-8cf2-3634943ac3e0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:12.191Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed1277a0-073d-4eb8-8cf2-3634943ac3e0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:12.191Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed1277a0-073d-4eb8-8cf2-3634943ac3e0","podName":"fail-bcffx-head-s6qj5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:46:12.191Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed1277a0-073d-4eb8-8cf2-3634943ac3e0","podName":"fail-bcffx-head-s6qj5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:46:12.191Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed1277a0-073d-4eb8-8cf2-3634943ac3e0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:12.191Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ed1277a0-073d-4eb8-8cf2-3634943ac3e0","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:14.582Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"8a9e6cee-b7e0-4b3e-935c-3d135b0ad6de","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:14.582Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"8a9e6cee-b7e0-4b3e-935c-3d135b0ad6de","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:14.582Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"8a9e6cee-b7e0-4b3e-935c-3d135b0ad6de","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:14.582Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"8a9e6cee-b7e0-4b3e-935c-3d135b0ad6de"}
{"level":"info","ts":"2026-09-18T09:46:14.582Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"8a9e6cee-b7e0-4b3e-935c-3d135b0ad6de","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:14.582Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"8a9e6cee-b7e0-4b3e-935c-3d135b0ad6de","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:17.583Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"b6a5d782-cc3b-4a58-816f-558e5a2cb12c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:17.583Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"b6a5d782-cc3b-4a58-816f-558e5a2cb12c","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:17.583Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"b6a5d782-cc3b-4a58-816f-558e5a2cb12c","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:17.583Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"b6a5d782-cc3b-4a58-816f-558e5a2cb12c"}
{"level":"info","ts":"2026-09-18T09:46:17.583Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"b6a5d782-cc3b-4a58-816f-558e5a2cb12c","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:17.583Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"b6a5d782-cc3b-4a58-816f-558e5a2cb12c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:18.387Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"aa8047ff-1c0e-49c3-b8e7-c05390659dee"}
{"level":"info","ts":"2026-09-18T09:46:18.388Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"aa8047ff-1c0e-49c3-b8e7-c05390659dee"}
{"level":"info","ts":"2026-09-18T09:46:18.388Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"aa8047ff-1c0e-49c3-b8e7-c05390659dee"}
{"level":"info","ts":"2026-09-18T09:46:18.388Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"aa8047ff-1c0e-49c3-b8e7-c05390659dee","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:18.388Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"aa8047ff-1c0e-49c3-b8e7-c05390659dee","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:18.388Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"aa8047ff-1c0e-49c3-b8e7-c05390659dee","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:18.388Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"aa8047ff-1c0e-49c3-b8e7-c05390659dee","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:18.388Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"aa8047ff-1c0e-49c3-b8e7-c05390659dee","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:18.388Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"aa8047ff-1c0e-49c3-b8e7-c05390659dee","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:18.388Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"aa8047ff-1c0e-49c3-b8e7-c05390659dee","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:18.388Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"aa8047ff-1c0e-49c3-b8e7-c05390659dee","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:18.388Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"aa8047ff-1c0e-49c3-b8e7-c05390659dee","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:18.388Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"aa8047ff-1c0e-49c3-b8e7-c05390659dee","podName":"fail-bcffx-head-s6qj5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:46:18.388Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"aa8047ff-1c0e-49c3-b8e7-c05390659dee","podName":"fail-bcffx-head-s6qj5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:46:18.388Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"aa8047ff-1c0e-49c3-b8e7-c05390659dee","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:18.388Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"aa8047ff-1c0e-49c3-b8e7-c05390659dee","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:19.393Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e0c5f228-d885-497a-b1ba-4e83ba1f5877"}
{"level":"info","ts":"2026-09-18T09:46:19.393Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e0c5f228-d885-497a-b1ba-4e83ba1f5877"}
{"level":"info","ts":"2026-09-18T09:46:19.393Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e0c5f228-d885-497a-b1ba-4e83ba1f5877"}
{"level":"info","ts":"2026-09-18T09:46:19.393Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e0c5f228-d885-497a-b1ba-4e83ba1f5877","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:19.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e0c5f228-d885-497a-b1ba-4e83ba1f5877","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:19.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e0c5f228-d885-497a-b1ba-4e83ba1f5877","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:19.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e0c5f228-d885-497a-b1ba-4e83ba1f5877","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:19.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e0c5f228-d885-497a-b1ba-4e83ba1f5877","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:19.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e0c5f228-d885-497a-b1ba-4e83ba1f5877","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:19.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e0c5f228-d885-497a-b1ba-4e83ba1f5877","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:19.394Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e0c5f228-d885-497a-b1ba-4e83ba1f5877","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:19.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e0c5f228-d885-497a-b1ba-4e83ba1f5877","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:19.394Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e0c5f228-d885-497a-b1ba-4e83ba1f5877","podName":"fail-bcffx-head-s6qj5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:46:19.394Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e0c5f228-d885-497a-b1ba-4e83ba1f5877","podName":"fail-bcffx-head-s6qj5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:46:19.394Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e0c5f228-d885-497a-b1ba-4e83ba1f5877","name":"fail-bcffx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:19.407Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"eec3fd23-1d10-4a94-92b3-615383f8d380","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:19.407Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"eec3fd23-1d10-4a94-92b3-615383f8d380","pod":"fail-bcffx-head-s6qj5","ip":"10.133.0.29"}
{"level":"info","ts":"2026-09-18T09:46:19.407Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"eec3fd23-1d10-4a94-92b3-615383f8d380","pod":"fail-bcffx-small-group-worker-dcw8j","ip":"10.132.0.31"}
{"level":"info","ts":"2026-09-18T09:46:19.407Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"eec3fd23-1d10-4a94-92b3-615383f8d380","rayCluster":"fail-bcffx","podIPs":["10.133.0.29","10.132.0.31"]}
{"level":"info","ts":"2026-09-18T09:46:19.407Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"7b64c50b-089d-4dd8-b28b-06aab8e09308","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:19.407Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"7b64c50b-089d-4dd8-b28b-06aab8e09308","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:19.407Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"7b64c50b-089d-4dd8-b28b-06aab8e09308","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:19.407Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"8cf58b49-352f-4a10-98b8-054e0cec1899","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:19.407Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"7b64c50b-089d-4dd8-b28b-06aab8e09308"}
{"level":"info","ts":"2026-09-18T09:46:19.407Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"7b64c50b-089d-4dd8-b28b-06aab8e09308","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:19.407Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"7b64c50b-089d-4dd8-b28b-06aab8e09308","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:19.407Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"7b64c50b-089d-4dd8-b28b-06aab8e09308","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:19.420Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"8cf58b49-352f-4a10-98b8-054e0cec1899","name":"fail-bcffx-head"}
{"level":"info","ts":"2026-09-18T09:46:19.421Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"710f8f23-d92b-4785-b0a7-6b9a5f03e1a8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:19.421Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"710f8f23-d92b-4785-b0a7-6b9a5f03e1a8","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:19.421Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"710f8f23-d92b-4785-b0a7-6b9a5f03e1a8","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:19.421Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"710f8f23-d92b-4785-b0a7-6b9a5f03e1a8"}
{"level":"info","ts":"2026-09-18T09:46:19.421Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"710f8f23-d92b-4785-b0a7-6b9a5f03e1a8","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:19.421Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"710f8f23-d92b-4785-b0a7-6b9a5f03e1a8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:19.432Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"8cf58b49-352f-4a10-98b8-054e0cec1899","name":"fail-bcffx-workers"}
{"level":"info","ts":"2026-09-18T09:46:19.432Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"8cf58b49-352f-4a10-98b8-054e0cec1899","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:20.584Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f9f00c45-7695-4bee-9f5e-4597fbdfe1f7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:20.584Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f9f00c45-7695-4bee-9f5e-4597fbdfe1f7","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:20.584Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f9f00c45-7695-4bee-9f5e-4597fbdfe1f7","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:20.584Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f9f00c45-7695-4bee-9f5e-4597fbdfe1f7"}
{"level":"info","ts":"2026-09-18T09:46:20.584Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f9f00c45-7695-4bee-9f5e-4597fbdfe1f7","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:20.584Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f9f00c45-7695-4bee-9f5e-4597fbdfe1f7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:21.207Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"bc619738-32c3-418e-aa13-25853c750f6f"}
{"level":"info","ts":"2026-09-18T09:46:21.207Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"bc619738-32c3-418e-aa13-25853c750f6f"}
{"level":"info","ts":"2026-09-18T09:46:21.207Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"bc619738-32c3-418e-aa13-25853c750f6f"}
{"level":"info","ts":"2026-09-18T09:46:21.207Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"bc619738-32c3-418e-aa13-25853c750f6f","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:21.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"bc619738-32c3-418e-aa13-25853c750f6f","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:21.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"bc619738-32c3-418e-aa13-25853c750f6f","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:21.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"bc619738-32c3-418e-aa13-25853c750f6f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:21.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"bc619738-32c3-418e-aa13-25853c750f6f","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:21.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"bc619738-32c3-418e-aa13-25853c750f6f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:21.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"bc619738-32c3-418e-aa13-25853c750f6f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:21.207Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"bc619738-32c3-418e-aa13-25853c750f6f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:21.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"bc619738-32c3-418e-aa13-25853c750f6f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:21.207Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"bc619738-32c3-418e-aa13-25853c750f6f","podName":"fail-bcffx-small-group-worker-dcw8j","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:46:21.207Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"bc619738-32c3-418e-aa13-25853c750f6f","podName":"fail-bcffx-small-group-worker-dcw8j","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:46:21.208Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"bc619738-32c3-418e-aa13-25853c750f6f","name":"fail-bcffx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:21.267Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6f0c0942-1e47-40c5-bd5d-78b9bda3531a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:21.268Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6f0c0942-1e47-40c5-bd5d-78b9bda3531a","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:21.268Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6f0c0942-1e47-40c5-bd5d-78b9bda3531a","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:21.268Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6f0c0942-1e47-40c5-bd5d-78b9bda3531a"}
{"level":"info","ts":"2026-09-18T09:46:21.268Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6f0c0942-1e47-40c5-bd5d-78b9bda3531a","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:21.268Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6f0c0942-1e47-40c5-bd5d-78b9bda3531a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:21.268Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6f0c0942-1e47-40c5-bd5d-78b9bda3531a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:21.268Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"25f90818-9ee4-49b3-b963-466bc532bd0f","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:21.268Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"25f90818-9ee4-49b3-b963-466bc532bd0f","pod":"fail-bcffx-head-s6qj5","ip":"10.133.0.29"}
{"level":"info","ts":"2026-09-18T09:46:21.268Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"1c2dded8-5624-462d-9ff2-149bb61dadd5","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:21.268Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"25f90818-9ee4-49b3-b963-466bc532bd0f","pod":"fail-bcffx-small-group-worker-dcw8j","ip":"10.132.0.31"}
{"level":"info","ts":"2026-09-18T09:46:21.268Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"25f90818-9ee4-49b3-b963-466bc532bd0f","rayCluster":"fail-bcffx","podIPs":["10.133.0.29","10.132.0.31"]}
{"level":"info","ts":"2026-09-18T09:46:21.281Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"fd3805fc-b973-4a1e-9ad8-c4af73522de1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:21.281Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"fd3805fc-b973-4a1e-9ad8-c4af73522de1","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:21.281Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"fd3805fc-b973-4a1e-9ad8-c4af73522de1","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:21.281Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"fd3805fc-b973-4a1e-9ad8-c4af73522de1"}
{"level":"info","ts":"2026-09-18T09:46:21.281Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"fd3805fc-b973-4a1e-9ad8-c4af73522de1","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:21.281Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"fd3805fc-b973-4a1e-9ad8-c4af73522de1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:21.284Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"1c2dded8-5624-462d-9ff2-149bb61dadd5","name":"fail-bcffx-head"}
{"level":"info","ts":"2026-09-18T09:46:21.296Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"1c2dded8-5624-462d-9ff2-149bb61dadd5","name":"fail-bcffx-workers"}
{"level":"info","ts":"2026-09-18T09:46:21.296Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"1c2dded8-5624-462d-9ff2-149bb61dadd5","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:21.407Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ba99c69e-4fb6-4882-8459-de8794797f39"}
{"level":"info","ts":"2026-09-18T09:46:21.407Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ba99c69e-4fb6-4882-8459-de8794797f39"}
{"level":"info","ts":"2026-09-18T09:46:21.407Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ba99c69e-4fb6-4882-8459-de8794797f39"}
{"level":"info","ts":"2026-09-18T09:46:21.407Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ba99c69e-4fb6-4882-8459-de8794797f39","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:21.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ba99c69e-4fb6-4882-8459-de8794797f39","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:21.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ba99c69e-4fb6-4882-8459-de8794797f39","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:21.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ba99c69e-4fb6-4882-8459-de8794797f39","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:21.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ba99c69e-4fb6-4882-8459-de8794797f39","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:21.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ba99c69e-4fb6-4882-8459-de8794797f39","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:21.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ba99c69e-4fb6-4882-8459-de8794797f39","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:21.407Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ba99c69e-4fb6-4882-8459-de8794797f39","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:21.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ba99c69e-4fb6-4882-8459-de8794797f39","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:21.408Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ba99c69e-4fb6-4882-8459-de8794797f39","podName":"fail-bcffx-small-group-worker-dcw8j","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:46:21.408Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ba99c69e-4fb6-4882-8459-de8794797f39","podName":"fail-bcffx-small-group-worker-dcw8j","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:46:21.408Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ba99c69e-4fb6-4882-8459-de8794797f39","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:21.408Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"ba99c69e-4fb6-4882-8459-de8794797f39","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:23.585Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"368ea06b-b719-4a8f-93c2-840d32f422f7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:23.585Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"368ea06b-b719-4a8f-93c2-840d32f422f7","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:23.585Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"368ea06b-b719-4a8f-93c2-840d32f422f7","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:23.585Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"368ea06b-b719-4a8f-93c2-840d32f422f7"}
{"level":"info","ts":"2026-09-18T09:46:23.585Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"368ea06b-b719-4a8f-93c2-840d32f422f7","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:23.585Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"368ea06b-b719-4a8f-93c2-840d32f422f7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:26.585Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"676d1539-e577-4a4d-b3b2-11eeab06178d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:26.586Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"676d1539-e577-4a4d-b3b2-11eeab06178d","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:26.586Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"676d1539-e577-4a4d-b3b2-11eeab06178d","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:26.586Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"676d1539-e577-4a4d-b3b2-11eeab06178d"}
{"level":"info","ts":"2026-09-18T09:46:26.586Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"676d1539-e577-4a4d-b3b2-11eeab06178d","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:26.586Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"676d1539-e577-4a4d-b3b2-11eeab06178d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:29.586Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"09e336bc-6fee-4be4-909f-cb6f2e759b66","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:29.586Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"09e336bc-6fee-4be4-909f-cb6f2e759b66","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:29.586Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"09e336bc-6fee-4be4-909f-cb6f2e759b66","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:29.586Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"09e336bc-6fee-4be4-909f-cb6f2e759b66"}
{"level":"info","ts":"2026-09-18T09:46:29.586Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"09e336bc-6fee-4be4-909f-cb6f2e759b66","RayCluster":"fail-bcffx","State":""}
{"level":"info","ts":"2026-09-18T09:46:29.586Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"09e336bc-6fee-4be4-909f-cb6f2e759b66","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:29.779Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"66b9552b-adde-407d-9bc4-3b2216c7c61b","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:29.779Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"66b9552b-adde-407d-9bc4-3b2216c7c61b","pod":"fail-bcffx-head-s6qj5","ip":"10.133.0.29"}
{"level":"info","ts":"2026-09-18T09:46:29.779Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"66b9552b-adde-407d-9bc4-3b2216c7c61b","pod":"fail-bcffx-small-group-worker-dcw8j","ip":"10.132.0.31"}
{"level":"info","ts":"2026-09-18T09:46:29.779Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"66b9552b-adde-407d-9bc4-3b2216c7c61b","rayCluster":"fail-bcffx","podIPs":["10.133.0.29","10.132.0.31"]}
{"level":"info","ts":"2026-09-18T09:46:30.421Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"dfa27005-5980-4939-b4ac-51c7899acf4a"}
{"level":"info","ts":"2026-09-18T09:46:30.421Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"dfa27005-5980-4939-b4ac-51c7899acf4a"}
{"level":"info","ts":"2026-09-18T09:46:30.421Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"dfa27005-5980-4939-b4ac-51c7899acf4a"}
{"level":"info","ts":"2026-09-18T09:46:30.421Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"dfa27005-5980-4939-b4ac-51c7899acf4a","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:30.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"dfa27005-5980-4939-b4ac-51c7899acf4a","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:30.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"dfa27005-5980-4939-b4ac-51c7899acf4a","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:30.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"dfa27005-5980-4939-b4ac-51c7899acf4a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:30.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"dfa27005-5980-4939-b4ac-51c7899acf4a","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:30.422Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"dfa27005-5980-4939-b4ac-51c7899acf4a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:30.422Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"dfa27005-5980-4939-b4ac-51c7899acf4a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:30.422Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"dfa27005-5980-4939-b4ac-51c7899acf4a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:30.422Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"dfa27005-5980-4939-b4ac-51c7899acf4a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:30.422Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"dfa27005-5980-4939-b4ac-51c7899acf4a","name":"fail-bcffx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:30Z","stateTransitionTimes":{"ready":"2026-09-18T09:46:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:46:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:30.437Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"e184650f-760c-407d-858c-56473cf98d7a","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:30.437Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"0a0ecb85-e67e-4362-b401-71ad7ec99f6c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:30.437Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"0a0ecb85-e67e-4362-b401-71ad7ec99f6c","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:30.438Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"0a0ecb85-e67e-4362-b401-71ad7ec99f6c","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:30.438Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"0a0ecb85-e67e-4362-b401-71ad7ec99f6c"}
{"level":"info","ts":"2026-09-18T09:46:30.438Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"0a0ecb85-e67e-4362-b401-71ad7ec99f6c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","dashboardURL":"fail-bcffx-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:30.438Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"0a0ecb85-e67e-4362-b401-71ad7ec99f6c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:46:30.438Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"75a13867-0c5a-4fea-b1bb-6c92e3b71bf0","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:30.438Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"75a13867-0c5a-4fea-b1bb-6c92e3b71bf0","pod":"fail-bcffx-head-s6qj5","ip":"10.133.0.29"}
{"level":"info","ts":"2026-09-18T09:46:30.438Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"75a13867-0c5a-4fea-b1bb-6c92e3b71bf0","pod":"fail-bcffx-small-group-worker-dcw8j","ip":"10.132.0.31"}
{"level":"info","ts":"2026-09-18T09:46:30.438Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"75a13867-0c5a-4fea-b1bb-6c92e3b71bf0","rayCluster":"fail-bcffx","podIPs":["10.133.0.29","10.132.0.31"]}
{"level":"info","ts":"2026-09-18T09:46:30.456Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"95fae7bd-0034-41d9-8a0d-9a735613147d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:30.456Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"95fae7bd-0034-41d9-8a0d-9a735613147d","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:30.456Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"95fae7bd-0034-41d9-8a0d-9a735613147d","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:30.456Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"95fae7bd-0034-41d9-8a0d-9a735613147d"}
{"level":"info","ts":"2026-09-18T09:46:30.456Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"95fae7bd-0034-41d9-8a0d-9a735613147d","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:46:30.481Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"e184650f-760c-407d-858c-56473cf98d7a","name":"fail-bcffx-head"}
{"level":"info","ts":"2026-09-18T09:46:30.482Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"95fae7bd-0034-41d9-8a0d-9a735613147d","JobId":"fail-25n69"}
{"level":"info","ts":"2026-09-18T09:46:30.493Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"e184650f-760c-407d-858c-56473cf98d7a","name":"fail-bcffx-workers"}
{"level":"info","ts":"2026-09-18T09:46:30.493Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"e184650f-760c-407d-858c-56473cf98d7a","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:31.303Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"d2cbb848-bed5-4539-a980-7abb7672e88f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:31.303Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"d2cbb848-bed5-4539-a980-7abb7672e88f","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:31.303Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"d2cbb848-bed5-4539-a980-7abb7672e88f","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:31.304Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"d2cbb848-bed5-4539-a980-7abb7672e88f"}
{"level":"info","ts":"2026-09-18T09:46:31.304Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"d2cbb848-bed5-4539-a980-7abb7672e88f","rayCluster":"fail-bcffx","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:46:31.308Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"d2cbb848-bed5-4539-a980-7abb7672e88f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","dashboardURL":"fail-bcffx-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:46:31Z"},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","dashboardURL":"fail-bcffx-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:30Z","stateTransitionTimes":{"ready":"2026-09-18T09:46:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:46:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:31.308Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"d2cbb848-bed5-4539-a980-7abb7672e88f","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:46:31.324Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"af0659d3-4abd-4259-8f61-7e116b4dd893","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:31.324Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"af0659d3-4abd-4259-8f61-7e116b4dd893","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:31.324Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"af0659d3-4abd-4259-8f61-7e116b4dd893","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:31.324Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"af0659d3-4abd-4259-8f61-7e116b4dd893"}
{"level":"info","ts":"2026-09-18T09:46:31.324Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"af0659d3-4abd-4259-8f61-7e116b4dd893","rayCluster":"fail-bcffx","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:46:31.327Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"af0659d3-4abd-4259-8f61-7e116b4dd893","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:46:31Z"},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","dashboardURL":"fail-bcffx-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:30Z","stateTransitionTimes":{"ready":"2026-09-18T09:46:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:46:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:46:31Z"},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","dashboardURL":"fail-bcffx-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:30Z","stateTransitionTimes":{"ready":"2026-09-18T09:46:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:46:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:32.439Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"eca55a61-30b7-4a7b-adc8-7da572c260ce"}
{"level":"info","ts":"2026-09-18T09:46:32.439Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"eca55a61-30b7-4a7b-adc8-7da572c260ce"}
{"level":"info","ts":"2026-09-18T09:46:32.439Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"eca55a61-30b7-4a7b-adc8-7da572c260ce"}
{"level":"info","ts":"2026-09-18T09:46:32.439Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"eca55a61-30b7-4a7b-adc8-7da572c260ce","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:32.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"eca55a61-30b7-4a7b-adc8-7da572c260ce","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:32.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"eca55a61-30b7-4a7b-adc8-7da572c260ce","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:32.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"eca55a61-30b7-4a7b-adc8-7da572c260ce","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:32.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"eca55a61-30b7-4a7b-adc8-7da572c260ce","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:32.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"eca55a61-30b7-4a7b-adc8-7da572c260ce","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:32.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"eca55a61-30b7-4a7b-adc8-7da572c260ce","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:32.439Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"eca55a61-30b7-4a7b-adc8-7da572c260ce","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:32.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"eca55a61-30b7-4a7b-adc8-7da572c260ce","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:32.440Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"eca55a61-30b7-4a7b-adc8-7da572c260ce","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:32.440Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"eca55a61-30b7-4a7b-adc8-7da572c260ce","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:32.587Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"eccaa13d-bd9c-4ba7-b479-82118d565982","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:32.587Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"eccaa13d-bd9c-4ba7-b479-82118d565982","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:32.588Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"eccaa13d-bd9c-4ba7-b479-82118d565982","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:32.588Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"eccaa13d-bd9c-4ba7-b479-82118d565982"}
{"level":"info","ts":"2026-09-18T09:46:32.588Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"eccaa13d-bd9c-4ba7-b479-82118d565982","rayCluster":"fail-bcffx","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:46:32.590Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"eccaa13d-bd9c-4ba7-b479-82118d565982","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:46:31Z"},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","dashboardURL":"fail-bcffx-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:30Z","stateTransitionTimes":{"ready":"2026-09-18T09:46:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:46:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:46:31Z"},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","dashboardURL":"fail-bcffx-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:30Z","stateTransitionTimes":{"ready":"2026-09-18T09:46:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:46:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:32.591Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"eccaa13d-bd9c-4ba7-b479-82118d565982","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:46:32.608Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"c2d59743-61b0-4fe5-a42e-802ddb2061fa","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:32.608Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"c2d59743-61b0-4fe5-a42e-802ddb2061fa","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:32.608Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"c2d59743-61b0-4fe5-a42e-802ddb2061fa","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:32.608Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"c2d59743-61b0-4fe5-a42e-802ddb2061fa"}
{"level":"info","ts":"2026-09-18T09:46:32.608Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"c2d59743-61b0-4fe5-a42e-802ddb2061fa","rayCluster":"fail-bcffx","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:46:32.610Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"c2d59743-61b0-4fe5-a42e-802ddb2061fa","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:46:31Z"},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","dashboardURL":"fail-bcffx-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:30Z","stateTransitionTimes":{"ready":"2026-09-18T09:46:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:46:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:46:31Z"},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","dashboardURL":"fail-bcffx-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:30Z","stateTransitionTimes":{"ready":"2026-09-18T09:46:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:46:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:35.608Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"939a92ef-9ea1-4649-9728-881137947528","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:35.608Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"939a92ef-9ea1-4649-9728-881137947528","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:46:35.608Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"939a92ef-9ea1-4649-9728-881137947528","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:35.608Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"939a92ef-9ea1-4649-9728-881137947528"}
{"level":"info","ts":"2026-09-18T09:46:35.608Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"939a92ef-9ea1-4649-9728-881137947528","rayCluster":"fail-bcffx","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:46:35.611Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"939a92ef-9ea1-4649-9728-881137947528","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:46:31Z"},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","dashboardURL":"fail-bcffx-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:30Z","stateTransitionTimes":{"ready":"2026-09-18T09:46:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:46:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:46:31Z","endTime":"2026-09-18T09:46:32Z"},"jobId":"fail-25n69","rayClusterName":"fail-bcffx","dashboardURL":"fail-bcffx-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"AppFailed","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:46:31,011\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-25n69: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:45:59Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:30Z","stateTransitionTimes":{"ready":"2026-09-18T09:46:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:46:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:35.611Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"939a92ef-9ea1-4649-9728-881137947528","old JobStatus":"RUNNING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-09-18T09:46:35.628Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"2602028a-d3ab-484d-9f5f-312b8ab22970","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:35.828Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"98afbfba-438e-48a5-8f85-692d04cf2fe1","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:35.828Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"98afbfba-438e-48a5-8f85-692d04cf2fe1","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:46:35.839Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"98afbfba-438e-48a5-8f85-692d04cf2fe1"}
{"level":"info","ts":"2026-09-18T09:46:35.839Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"98afbfba-438e-48a5-8f85-692d04cf2fe1","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:46:35.839Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"98afbfba-438e-48a5-8f85-692d04cf2fe1","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:46:35.839Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"98afbfba-438e-48a5-8f85-692d04cf2fe1","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:35.850Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"6e35d632-459b-484b-8599-b349fe492f4f","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:35.850Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"6e35d632-459b-484b-8599-b349fe492f4f"}
{"level":"info","ts":"2026-09-18T09:46:35.850Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"6e35d632-459b-484b-8599-b349fe492f4f","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:46:35.850Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"6e35d632-459b-484b-8599-b349fe492f4f","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-rft6g","rayClusterName":"stop-nkvp5","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:46:35.850Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"6e35d632-459b-484b-8599-b349fe492f4f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:35.856Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"6e35d632-459b-484b-8599-b349fe492f4f","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T09:46:35.856Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"6e35d632-459b-484b-8599-b349fe492f4f"}
{"level":"error","ts":"2026-09-18T09:46:35.856Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"6e35d632-459b-484b-8599-b349fe492f4f","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:35.856Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"a5cd0153-68cf-4d01-bdc4-d32c4ed43106","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:35.856Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"a5cd0153-68cf-4d01-bdc4-d32c4ed43106","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.856Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"a5cd0153-68cf-4d01-bdc4-d32c4ed43106","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:35.856Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"a5cd0153-68cf-4d01-bdc4-d32c4ed43106","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:35.861Z","logger":"raycluster-resource","msg":"default","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.861Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-zllpq","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:46:35.873Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"a5cd0153-68cf-4d01-bdc4-d32c4ed43106","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:35.873Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"a5cd0153-68cf-4d01-bdc4-d32c4ed43106"}
{"level":"info","ts":"2026-09-18T09:46:35.873Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"a5cd0153-68cf-4d01-bdc4-d32c4ed43106","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:35.873Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"a5cd0153-68cf-4d01-bdc4-d32c4ed43106","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"7ef6a417-3329-442e-b657-f61fcebfe58a","namespacedName":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"7ef6a417-3329-442e-b657-f61fcebfe58a","mode":"IntegratedOAuth","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"7ef6a417-3329-442e-b657-f61fcebfe58a","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"7ef6a417-3329-442e-b657-f61fcebfe58a","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f0132852-90cc-49d0-9562-61daf14443c8","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f0132852-90cc-49d0-9562-61daf14443c8","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"065f788d-87b9-45c9-9ffa-c3bf93ceaf86"}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"065f788d-87b9-45c9-9ffa-c3bf93ceaf86"}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"065f788d-87b9-45c9-9ffa-c3bf93ceaf86"}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f665b96d-bc97-4572-b0be-bcb0337ed2d1","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"241ff701-d602-4a02-952b-55718f38dcef","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"241ff701-d602-4a02-952b-55718f38dcef","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"241ff701-d602-4a02-952b-55718f38dcef","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"241ff701-d602-4a02-952b-55718f38dcef"}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"241ff701-d602-4a02-952b-55718f38dcef","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"241ff701-d602-4a02-952b-55718f38dcef","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"46a8064e-f737-4bf5-a8f8-e08d8e67f564","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"46a8064e-f737-4bf5-a8f8-e08d8e67f564","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"46a8064e-f737-4bf5-a8f8-e08d8e67f564","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"46a8064e-f737-4bf5-a8f8-e08d8e67f564"}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"46a8064e-f737-4bf5-a8f8-e08d8e67f564","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:35.874Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"46a8064e-f737-4bf5-a8f8-e08d8e67f564","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:46:35.880Z","logger":"raycluster-resource","msg":"default","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.880Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-zllpq","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:46:35.881Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"065f788d-87b9-45c9-9ffa-c3bf93ceaf86","name":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:35.882Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"065f788d-87b9-45c9-9ffa-c3bf93ceaf86","error":"head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"error","ts":"2026-09-18T09:46:35.882Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"065f788d-87b9-45c9-9ffa-c3bf93ceaf86","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:35.882Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"065f788d-87b9-45c9-9ffa-c3bf93ceaf86","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"info","ts":"2026-09-18T09:46:35.882Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"065f788d-87b9-45c9-9ffa-c3bf93ceaf86","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:35.882Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"065f788d-87b9-45c9-9ffa-c3bf93ceaf86","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:35.882Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"065f788d-87b9-45c9-9ffa-c3bf93ceaf86","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:35.882Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"065f788d-87b9-45c9-9ffa-c3bf93ceaf86","name":"stop-zllpq","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:35.885Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f0132852-90cc-49d0-9562-61daf14443c8","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.885Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f0132852-90cc-49d0-9562-61daf14443c8","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.892Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"fb960b54-46ac-4bdd-8a73-1b3f0d437cfc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:35.892Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"fb960b54-46ac-4bdd-8a73-1b3f0d437cfc","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.892Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"fb960b54-46ac-4bdd-8a73-1b3f0d437cfc","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:35.892Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"fb960b54-46ac-4bdd-8a73-1b3f0d437cfc"}
{"level":"info","ts":"2026-09-18T09:46:35.892Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"fb960b54-46ac-4bdd-8a73-1b3f0d437cfc","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:35.892Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"fb960b54-46ac-4bdd-8a73-1b3f0d437cfc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:46:35.893Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f665b96d-bc97-4572-b0be-bcb0337ed2d1","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.894Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"d2d7b69c-869b-41e9-88b6-5c36156a97b4","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.895Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"065f788d-87b9-45c9-9ffa-c3bf93ceaf86","name":"stop-zllpq","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-zllpq\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-q2lj2","name":"stop-zllpq"}}
{"level":"info","ts":"2026-09-18T09:46:35.895Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"065f788d-87b9-45c9-9ffa-c3bf93ceaf86"}
{"level":"error","ts":"2026-09-18T09:46:35.895Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"065f788d-87b9-45c9-9ffa-c3bf93ceaf86","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:35.896Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"a6848fb1-5b64-4523-8ee1-8c8fb4febbdc"}
{"level":"info","ts":"2026-09-18T09:46:35.896Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"a6848fb1-5b64-4523-8ee1-8c8fb4febbdc"}
{"level":"info","ts":"2026-09-18T09:46:35.896Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"a6848fb1-5b64-4523-8ee1-8c8fb4febbdc"}
{"level":"info","ts":"2026-09-18T09:46:35.896Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"a6848fb1-5b64-4523-8ee1-8c8fb4febbdc","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:35.896Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"a6848fb1-5b64-4523-8ee1-8c8fb4febbdc","error":"head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"error","ts":"2026-09-18T09:46:35.896Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"a6848fb1-5b64-4523-8ee1-8c8fb4febbdc","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:35.896Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"a6848fb1-5b64-4523-8ee1-8c8fb4febbdc","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"info","ts":"2026-09-18T09:46:35.896Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"a6848fb1-5b64-4523-8ee1-8c8fb4febbdc","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:35.896Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"a6848fb1-5b64-4523-8ee1-8c8fb4febbdc","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:35.896Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"a6848fb1-5b64-4523-8ee1-8c8fb4febbdc","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:35.896Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"a6848fb1-5b64-4523-8ee1-8c8fb4febbdc","name":"stop-zllpq","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:35.897Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"7ef6a417-3329-442e-b657-f61fcebfe58a","name":"stop-zllpq-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:46:35.898Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f0132852-90cc-49d0-9562-61daf14443c8","rayCluster":"stop-zllpq","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T09:46:35.898Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f0132852-90cc-49d0-9562-61daf14443c8","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.966Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f0132852-90cc-49d0-9562-61daf14443c8","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.966Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"a6848fb1-5b64-4523-8ee1-8c8fb4febbdc"}
{"level":"error","ts":"2026-09-18T09:46:35.966Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"a6848fb1-5b64-4523-8ee1-8c8fb4febbdc","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:35.966Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"391a7e08-4f6e-4cdf-b321-c364a9f80d74"}
{"level":"info","ts":"2026-09-18T09:46:35.966Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"391a7e08-4f6e-4cdf-b321-c364a9f80d74"}
{"level":"info","ts":"2026-09-18T09:46:35.966Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"391a7e08-4f6e-4cdf-b321-c364a9f80d74"}
{"level":"info","ts":"2026-09-18T09:46:35.966Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"391a7e08-4f6e-4cdf-b321-c364a9f80d74","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:35.966Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"391a7e08-4f6e-4cdf-b321-c364a9f80d74","error":"head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"info","ts":"2026-09-18T09:46:35.966Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"d2d7b69c-869b-41e9-88b6-5c36156a97b4","name":"stop-zllpq-head"}
{"level":"error","ts":"2026-09-18T09:46:35.966Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"391a7e08-4f6e-4cdf-b321-c364a9f80d74","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:35.966Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"391a7e08-4f6e-4cdf-b321-c364a9f80d74","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"info","ts":"2026-09-18T09:46:35.966Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"391a7e08-4f6e-4cdf-b321-c364a9f80d74","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:35.966Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"391a7e08-4f6e-4cdf-b321-c364a9f80d74","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:35.966Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"391a7e08-4f6e-4cdf-b321-c364a9f80d74","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:35.966Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"391a7e08-4f6e-4cdf-b321-c364a9f80d74","name":"stop-zllpq","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:35.967Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d7f5d5da-4296-4d29-a340-dfd94cc29001","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:35.967Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d7f5d5da-4296-4d29-a340-dfd94cc29001","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.967Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d7f5d5da-4296-4d29-a340-dfd94cc29001","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:35.967Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d7f5d5da-4296-4d29-a340-dfd94cc29001"}
{"level":"info","ts":"2026-09-18T09:46:35.967Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d7f5d5da-4296-4d29-a340-dfd94cc29001","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:35.967Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d7f5d5da-4296-4d29-a340-dfd94cc29001","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:35.967Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d7f5d5da-4296-4d29-a340-dfd94cc29001","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:35.976Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"391a7e08-4f6e-4cdf-b321-c364a9f80d74","name":"stop-zllpq","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-zllpq\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-q2lj2","name":"stop-zllpq"}}
{"level":"info","ts":"2026-09-18T09:46:35.976Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"391a7e08-4f6e-4cdf-b321-c364a9f80d74"}
{"level":"error","ts":"2026-09-18T09:46:35.976Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"391a7e08-4f6e-4cdf-b321-c364a9f80d74","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:35.977Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d50583c4-e03b-4740-bec5-3598fe1a1db4"}
{"level":"info","ts":"2026-09-18T09:46:35.977Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d50583c4-e03b-4740-bec5-3598fe1a1db4"}
{"level":"info","ts":"2026-09-18T09:46:35.977Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d50583c4-e03b-4740-bec5-3598fe1a1db4"}
{"level":"info","ts":"2026-09-18T09:46:35.977Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d50583c4-e03b-4740-bec5-3598fe1a1db4","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:35.977Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d50583c4-e03b-4740-bec5-3598fe1a1db4","error":"head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"error","ts":"2026-09-18T09:46:35.977Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d50583c4-e03b-4740-bec5-3598fe1a1db4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:35.977Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d50583c4-e03b-4740-bec5-3598fe1a1db4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"info","ts":"2026-09-18T09:46:35.977Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d50583c4-e03b-4740-bec5-3598fe1a1db4","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:35.977Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d50583c4-e03b-4740-bec5-3598fe1a1db4","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:35.977Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d50583c4-e03b-4740-bec5-3598fe1a1db4","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:35.977Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d50583c4-e03b-4740-bec5-3598fe1a1db4"}
{"level":"error","ts":"2026-09-18T09:46:35.977Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d50583c4-e03b-4740-bec5-3598fe1a1db4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:35.978Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f0132852-90cc-49d0-9562-61daf14443c8","rayCluster":"stop-zllpq","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:46:35.978Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"7ef6a417-3329-442e-b657-f61fcebfe58a","name":"test-ns-q2lj2-stop-zllpq","operation":"created"}
{"level":"info","ts":"2026-09-18T09:46:35.981Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"d2d7b69c-869b-41e9-88b6-5c36156a97b4","name":"stop-zllpq-workers"}
{"level":"info","ts":"2026-09-18T09:46:35.981Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"d2d7b69c-869b-41e9-88b6-5c36156a97b4","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.981Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"b9b1ab98-4a08-4e77-b5a0-a6c834a85ac8","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.982Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"97695427-caa7-4116-895d-030355475d4b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:35.982Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"97695427-caa7-4116-895d-030355475d4b","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:35.982Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"97695427-caa7-4116-895d-030355475d4b","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:35.982Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"97695427-caa7-4116-895d-030355475d4b"}
{"level":"info","ts":"2026-09-18T09:46:35.982Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"97695427-caa7-4116-895d-030355475d4b","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:35.982Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"97695427-caa7-4116-895d-030355475d4b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:36.018Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6af0b9d1-09db-4025-b6d4-650a5fcbc83d"}
{"level":"info","ts":"2026-09-18T09:46:36.018Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6af0b9d1-09db-4025-b6d4-650a5fcbc83d"}
{"level":"info","ts":"2026-09-18T09:46:36.018Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6af0b9d1-09db-4025-b6d4-650a5fcbc83d"}
{"level":"info","ts":"2026-09-18T09:46:36.018Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6af0b9d1-09db-4025-b6d4-650a5fcbc83d","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:36.018Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6af0b9d1-09db-4025-b6d4-650a5fcbc83d","error":"head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"error","ts":"2026-09-18T09:46:36.018Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6af0b9d1-09db-4025-b6d4-650a5fcbc83d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:36.018Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6af0b9d1-09db-4025-b6d4-650a5fcbc83d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"info","ts":"2026-09-18T09:46:36.018Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6af0b9d1-09db-4025-b6d4-650a5fcbc83d","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:36.018Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6af0b9d1-09db-4025-b6d4-650a5fcbc83d","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:36.018Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6af0b9d1-09db-4025-b6d4-650a5fcbc83d","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:36.018Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6af0b9d1-09db-4025-b6d4-650a5fcbc83d"}
{"level":"error","ts":"2026-09-18T09:46:36.018Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6af0b9d1-09db-4025-b6d4-650a5fcbc83d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:36.066Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f0132852-90cc-49d0-9562-61daf14443c8","rayCluster":"stop-zllpq","podIPs":[]}
{"level":"error","ts":"2026-09-18T09:46:36.066Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f0132852-90cc-49d0-9562-61daf14443c8","error":"Certificate.cert-manager.io \"ray-worker-cert-stop-zllpq\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:36.066Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f0132852-90cc-49d0-9562-61daf14443c8"}
{"level":"error","ts":"2026-09-18T09:46:36.066Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f0132852-90cc-49d0-9562-61daf14443c8","error":"Certificate.cert-manager.io \"ray-worker-cert-stop-zllpq\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:36.066Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"0ff8325c-2018-428b-9c80-23f122fd14dc","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:36.067Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"7ef6a417-3329-442e-b657-f61fcebfe58a","name":"kube-rbac-proxy-config-stop-zllpq","operation":"created"}
{"level":"info","ts":"2026-09-18T09:46:36.073Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"b9b1ab98-4a08-4e77-b5a0-a6c834a85ac8","name":"stop-zllpq-head"}
{"level":"error","ts":"2026-09-18T09:46:36.083Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"7ef6a417-3329-442e-b657-f61fcebfe58a","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-zllpq\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:36.083Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"7ef6a417-3329-442e-b657-f61fcebfe58a"}
{"level":"error","ts":"2026-09-18T09:46:36.083Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"7ef6a417-3329-442e-b657-f61fcebfe58a","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-zllpq\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:36.083Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"41e39b94-5ded-4952-a676-ed3c268579d2","namespacedName":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:36.083Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"41e39b94-5ded-4952-a676-ed3c268579d2","mode":"IntegratedOAuth","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:36.083Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"41e39b94-5ded-4952-a676-ed3c268579d2","cluster":"stop-zllpq"}
{"level":"error","ts":"2026-09-18T09:46:36.085Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"0ff8325c-2018-428b-9c80-23f122fd14dc","error":"certificates.cert-manager.io \"ray-worker-cert-stop-zllpq\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:46:36.085Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"0ff8325c-2018-428b-9c80-23f122fd14dc","error":"certificates.cert-manager.io \"ray-worker-cert-stop-zllpq\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:36.085Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"0ff8325c-2018-428b-9c80-23f122fd14dc"}
{"level":"error","ts":"2026-09-18T09:46:36.085Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"0ff8325c-2018-428b-9c80-23f122fd14dc","error":"certificates.cert-manager.io \"ray-worker-cert-stop-zllpq\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:36.085Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"43b22c3f-ef7b-4ec6-b722-cd9612ceb12f","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:36.085Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"43b22c3f-ef7b-4ec6-b722-cd9612ceb12f"}
{"level":"info","ts":"2026-09-18T09:46:36.090Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"b9b1ab98-4a08-4e77-b5a0-a6c834a85ac8","name":"stop-zllpq-workers"}
{"level":"info","ts":"2026-09-18T09:46:36.090Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"b9b1ab98-4a08-4e77-b5a0-a6c834a85ac8","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:36.095Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"7238da32-774f-49d6-872d-1123c8c76da6","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:36.095Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"7238da32-774f-49d6-872d-1123c8c76da6"}
{"level":"info","ts":"2026-09-18T09:46:36.098Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"498b6b1e-069b-4561-8b4c-7bc201fb1c57"}
{"level":"info","ts":"2026-09-18T09:46:36.098Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"498b6b1e-069b-4561-8b4c-7bc201fb1c57"}
{"level":"info","ts":"2026-09-18T09:46:36.098Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"498b6b1e-069b-4561-8b4c-7bc201fb1c57"}
{"level":"info","ts":"2026-09-18T09:46:36.098Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"498b6b1e-069b-4561-8b4c-7bc201fb1c57","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:36.098Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"498b6b1e-069b-4561-8b4c-7bc201fb1c57","error":"head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"error","ts":"2026-09-18T09:46:36.098Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"498b6b1e-069b-4561-8b4c-7bc201fb1c57","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:36.099Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"498b6b1e-069b-4561-8b4c-7bc201fb1c57","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"info","ts":"2026-09-18T09:46:36.099Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"498b6b1e-069b-4561-8b4c-7bc201fb1c57","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:36.099Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"498b6b1e-069b-4561-8b4c-7bc201fb1c57","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:36.099Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"498b6b1e-069b-4561-8b4c-7bc201fb1c57","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:36.099Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"498b6b1e-069b-4561-8b4c-7bc201fb1c57"}
{"level":"error","ts":"2026-09-18T09:46:36.099Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"498b6b1e-069b-4561-8b4c-7bc201fb1c57","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:36.099Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"a7f6f2c4-62cc-43a6-94cf-0b4b496622ed","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:36.099Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"a7f6f2c4-62cc-43a6-94cf-0b4b496622ed"}
{"level":"info","ts":"2026-09-18T09:46:36.099Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"ca71aa6a-34c9-4e28-badf-600625826d63","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:36.099Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"ca71aa6a-34c9-4e28-badf-600625826d63","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:36.099Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"ca71aa6a-34c9-4e28-badf-600625826d63","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:36.099Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"d1908fb9-301b-43bd-a5d7-8478884a8e29","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:36.099Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"ca71aa6a-34c9-4e28-badf-600625826d63"}
{"level":"info","ts":"2026-09-18T09:46:36.099Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"ca71aa6a-34c9-4e28-badf-600625826d63","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:36.099Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"ca71aa6a-34c9-4e28-badf-600625826d63","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:36.099Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"ca71aa6a-34c9-4e28-badf-600625826d63","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:36.100Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"41e39b94-5ded-4952-a676-ed3c268579d2","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:36.100Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"3ac40fc7-2d06-4bf0-a661-577696ac1025","namespacedName":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:36.100Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"3ac40fc7-2d06-4bf0-a661-577696ac1025","mode":"IntegratedOAuth","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:36.100Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"3ac40fc7-2d06-4bf0-a661-577696ac1025","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:36.115Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"3ac40fc7-2d06-4bf0-a661-577696ac1025","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:36.166Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"d1908fb9-301b-43bd-a5d7-8478884a8e29","name":"stop-zllpq-head"}
{"level":"info","ts":"2026-09-18T09:46:36.166Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"5009aba5-a8fd-4030-8b24-bfea518931bf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:36.166Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"5009aba5-a8fd-4030-8b24-bfea518931bf","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:36.166Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"5009aba5-a8fd-4030-8b24-bfea518931bf","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:36.167Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"5009aba5-a8fd-4030-8b24-bfea518931bf"}
{"level":"info","ts":"2026-09-18T09:46:36.167Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"5009aba5-a8fd-4030-8b24-bfea518931bf","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:36.167Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"5009aba5-a8fd-4030-8b24-bfea518931bf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:36.178Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"d1908fb9-301b-43bd-a5d7-8478884a8e29","name":"stop-zllpq-workers"}
{"level":"info","ts":"2026-09-18T09:46:36.178Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"d1908fb9-301b-43bd-a5d7-8478884a8e29","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:36.260Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"64ea1b38-469b-40c1-bc35-3b77072a4fbd"}
{"level":"info","ts":"2026-09-18T09:46:36.260Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"64ea1b38-469b-40c1-bc35-3b77072a4fbd"}
{"level":"info","ts":"2026-09-18T09:46:36.260Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"64ea1b38-469b-40c1-bc35-3b77072a4fbd"}
{"level":"info","ts":"2026-09-18T09:46:36.260Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"64ea1b38-469b-40c1-bc35-3b77072a4fbd","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:36.260Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"64ea1b38-469b-40c1-bc35-3b77072a4fbd","error":"head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"error","ts":"2026-09-18T09:46:36.260Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"64ea1b38-469b-40c1-bc35-3b77072a4fbd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:36.260Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"64ea1b38-469b-40c1-bc35-3b77072a4fbd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"info","ts":"2026-09-18T09:46:36.260Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"64ea1b38-469b-40c1-bc35-3b77072a4fbd","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:36.260Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"64ea1b38-469b-40c1-bc35-3b77072a4fbd","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:36.260Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"64ea1b38-469b-40c1-bc35-3b77072a4fbd","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:36.260Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"64ea1b38-469b-40c1-bc35-3b77072a4fbd"}
{"level":"error","ts":"2026-09-18T09:46:36.260Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"64ea1b38-469b-40c1-bc35-3b77072a4fbd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:36.580Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"8490e60e-d889-462d-b232-9ca70f0249dd"}
{"level":"info","ts":"2026-09-18T09:46:36.580Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"8490e60e-d889-462d-b232-9ca70f0249dd"}
{"level":"info","ts":"2026-09-18T09:46:36.581Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"8490e60e-d889-462d-b232-9ca70f0249dd"}
{"level":"info","ts":"2026-09-18T09:46:36.581Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"8490e60e-d889-462d-b232-9ca70f0249dd","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:36.581Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"8490e60e-d889-462d-b232-9ca70f0249dd","error":"head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"error","ts":"2026-09-18T09:46:36.581Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"8490e60e-d889-462d-b232-9ca70f0249dd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:36.581Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"8490e60e-d889-462d-b232-9ca70f0249dd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"info","ts":"2026-09-18T09:46:36.581Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"8490e60e-d889-462d-b232-9ca70f0249dd","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:36.581Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"8490e60e-d889-462d-b232-9ca70f0249dd","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:36.581Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"8490e60e-d889-462d-b232-9ca70f0249dd","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:36.581Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"8490e60e-d889-462d-b232-9ca70f0249dd"}
{"level":"error","ts":"2026-09-18T09:46:36.581Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"8490e60e-d889-462d-b232-9ca70f0249dd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:37.222Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"04b187cb-e3c9-4e40-a2ed-760a3d97033e"}
{"level":"info","ts":"2026-09-18T09:46:37.222Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"04b187cb-e3c9-4e40-a2ed-760a3d97033e"}
{"level":"info","ts":"2026-09-18T09:46:37.222Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"04b187cb-e3c9-4e40-a2ed-760a3d97033e"}
{"level":"info","ts":"2026-09-18T09:46:37.222Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"04b187cb-e3c9-4e40-a2ed-760a3d97033e","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:37.222Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"04b187cb-e3c9-4e40-a2ed-760a3d97033e","error":"head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"error","ts":"2026-09-18T09:46:37.222Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"04b187cb-e3c9-4e40-a2ed-760a3d97033e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:37.222Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"04b187cb-e3c9-4e40-a2ed-760a3d97033e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found"}
{"level":"info","ts":"2026-09-18T09:46:37.222Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"04b187cb-e3c9-4e40-a2ed-760a3d97033e","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:37.222Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"04b187cb-e3c9-4e40-a2ed-760a3d97033e","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:37.222Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"04b187cb-e3c9-4e40-a2ed-760a3d97033e","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"stop-zllpq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:46:37.222Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"04b187cb-e3c9-4e40-a2ed-760a3d97033e"}
{"level":"error","ts":"2026-09-18T09:46:37.222Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"04b187cb-e3c9-4e40-a2ed-760a3d97033e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-zllpq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:38.503Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e"}
{"level":"info","ts":"2026-09-18T09:46:38.503Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e"}
{"level":"info","ts":"2026-09-18T09:46:38.503Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e"}
{"level":"info","ts":"2026-09-18T09:46:38.503Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:38.503Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e"}
{"level":"info","ts":"2026-09-18T09:46:38.503Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:46:38.503Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e"}
{"level":"info","ts":"2026-09-18T09:46:38.503Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","mode":"IntegratedOAuth","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:38.503Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","cluster":"stop-zllpq","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:46:38.503Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","cluster":"stop-zllpq","authType":"IntegratedOAuth","serviceAccount":"stop-zllpq-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:46:38.503Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"stop-zllpq","ray.io/group":"headgroup","ray.io/identifier":"stop-zllpq-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:46:38.503Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:46:38.503Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:46:38.503Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:46:38.503Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","enabled":true}
{"level":"info","ts":"2026-09-18T09:46:38.523Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","name":"stop-zllpq-head-ctt42"}
{"level":"info","ts":"2026-09-18T09:46:38.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:38.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:38.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:46:38.523Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:46:38.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:38.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:46:38.524Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e"}
{"level":"info","ts":"2026-09-18T09:46:38.524Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","nodeType":"worker","rayStartParams":{"address":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:46:38.524Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","rayStartCmd":"ray start  --address=stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:46:38.524Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:46:38.524Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","enabled":true}
{"level":"info","ts":"2026-09-18T09:46:38.542Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","name":"stop-zllpq-small-group-worker-bfxtm"}
{"level":"info","ts":"2026-09-18T09:46:38.542Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:38.543Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d974bd73-12cc-4bdb-be55-028e2d16f30e","name":"stop-zllpq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a"}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a"}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a"}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a","name":"stop-zllpq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"5844166d-3456-4de3-8b66-face78473e75","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:38.564Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"5844166d-3456-4de3-8b66-face78473e75","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:38.565Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"48af1a54-918a-487f-8c84-62c7c6006127","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:38.565Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"48af1a54-918a-487f-8c84-62c7c6006127","rayCluster":"stop-zllpq","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:46:38.565Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"6b24e234-c6bf-4803-8049-b0bd83cc6463","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:38.565Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"5844166d-3456-4de3-8b66-face78473e75","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:38.566Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"5844166d-3456-4de3-8b66-face78473e75"}
{"level":"info","ts":"2026-09-18T09:46:38.566Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"5844166d-3456-4de3-8b66-face78473e75","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:38.566Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"5844166d-3456-4de3-8b66-face78473e75","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:38.566Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"5844166d-3456-4de3-8b66-face78473e75","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:38.574Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a","name":"stop-zllpq","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-zllpq\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-q2lj2","name":"stop-zllpq"}}
{"level":"info","ts":"2026-09-18T09:46:38.574Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a"}
{"level":"error","ts":"2026-09-18T09:46:38.574Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"95e62cfc-33ee-4ef6-a7df-de2954c6fc7a","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-zllpq\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:46:38.574Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f926fd2a-2da9-451c-bac5-c1429d11fe6b"}
{"level":"info","ts":"2026-09-18T09:46:38.574Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f926fd2a-2da9-451c-bac5-c1429d11fe6b"}
{"level":"info","ts":"2026-09-18T09:46:38.574Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f926fd2a-2da9-451c-bac5-c1429d11fe6b"}
{"level":"info","ts":"2026-09-18T09:46:38.574Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f926fd2a-2da9-451c-bac5-c1429d11fe6b","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:38.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f926fd2a-2da9-451c-bac5-c1429d11fe6b","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:38.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f926fd2a-2da9-451c-bac5-c1429d11fe6b","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:38.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f926fd2a-2da9-451c-bac5-c1429d11fe6b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:38.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f926fd2a-2da9-451c-bac5-c1429d11fe6b","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:38.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f926fd2a-2da9-451c-bac5-c1429d11fe6b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:38.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f926fd2a-2da9-451c-bac5-c1429d11fe6b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:38.575Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f926fd2a-2da9-451c-bac5-c1429d11fe6b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:38.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f926fd2a-2da9-451c-bac5-c1429d11fe6b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:38.575Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f926fd2a-2da9-451c-bac5-c1429d11fe6b","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:38.575Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f926fd2a-2da9-451c-bac5-c1429d11fe6b","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:38.575Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f926fd2a-2da9-451c-bac5-c1429d11fe6b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:38.575Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f926fd2a-2da9-451c-bac5-c1429d11fe6b","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:38.580Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"809242a2-7175-46d8-b2d4-ce9dd7c529e6"}
{"level":"info","ts":"2026-09-18T09:46:38.580Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"809242a2-7175-46d8-b2d4-ce9dd7c529e6"}
{"level":"info","ts":"2026-09-18T09:46:38.580Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"809242a2-7175-46d8-b2d4-ce9dd7c529e6"}
{"level":"info","ts":"2026-09-18T09:46:38.580Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"809242a2-7175-46d8-b2d4-ce9dd7c529e6","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:38.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"809242a2-7175-46d8-b2d4-ce9dd7c529e6","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:38.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"809242a2-7175-46d8-b2d4-ce9dd7c529e6","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:38.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"809242a2-7175-46d8-b2d4-ce9dd7c529e6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:38.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"809242a2-7175-46d8-b2d4-ce9dd7c529e6","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:38.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"809242a2-7175-46d8-b2d4-ce9dd7c529e6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:38.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"809242a2-7175-46d8-b2d4-ce9dd7c529e6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:38.581Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"809242a2-7175-46d8-b2d4-ce9dd7c529e6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:38.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"809242a2-7175-46d8-b2d4-ce9dd7c529e6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:38.581Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"809242a2-7175-46d8-b2d4-ce9dd7c529e6","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:38.581Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"809242a2-7175-46d8-b2d4-ce9dd7c529e6","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:38.581Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"809242a2-7175-46d8-b2d4-ce9dd7c529e6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:38.581Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"809242a2-7175-46d8-b2d4-ce9dd7c529e6","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:38.582Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"6b24e234-c6bf-4803-8049-b0bd83cc6463","name":"stop-zllpq-head"}
{"level":"info","ts":"2026-09-18T09:46:38.584Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"6c5368c8-28de-448a-a185-9280460f94c0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:38.584Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"6c5368c8-28de-448a-a185-9280460f94c0","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:38.584Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"6c5368c8-28de-448a-a185-9280460f94c0","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:38.584Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"6c5368c8-28de-448a-a185-9280460f94c0"}
{"level":"info","ts":"2026-09-18T09:46:38.584Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"6c5368c8-28de-448a-a185-9280460f94c0","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:38.584Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"6c5368c8-28de-448a-a185-9280460f94c0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:38.586Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c9cf25db-22c5-4b40-8d00-a0b4982645e6"}
{"level":"info","ts":"2026-09-18T09:46:38.586Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c9cf25db-22c5-4b40-8d00-a0b4982645e6"}
{"level":"info","ts":"2026-09-18T09:46:38.586Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c9cf25db-22c5-4b40-8d00-a0b4982645e6"}
{"level":"info","ts":"2026-09-18T09:46:38.586Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c9cf25db-22c5-4b40-8d00-a0b4982645e6","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:38.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c9cf25db-22c5-4b40-8d00-a0b4982645e6","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:38.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c9cf25db-22c5-4b40-8d00-a0b4982645e6","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:38.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c9cf25db-22c5-4b40-8d00-a0b4982645e6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:38.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c9cf25db-22c5-4b40-8d00-a0b4982645e6","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:38.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c9cf25db-22c5-4b40-8d00-a0b4982645e6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:38.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c9cf25db-22c5-4b40-8d00-a0b4982645e6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:38.586Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c9cf25db-22c5-4b40-8d00-a0b4982645e6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:38.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c9cf25db-22c5-4b40-8d00-a0b4982645e6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:38.586Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c9cf25db-22c5-4b40-8d00-a0b4982645e6","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:38.586Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c9cf25db-22c5-4b40-8d00-a0b4982645e6","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:38.586Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c9cf25db-22c5-4b40-8d00-a0b4982645e6","name":"stop-zllpq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:38.597Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"6b24e234-c6bf-4803-8049-b0bd83cc6463","name":"stop-zllpq-workers"}
{"level":"info","ts":"2026-09-18T09:46:38.597Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"6b24e234-c6bf-4803-8049-b0bd83cc6463","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:38.600Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"e8f00739-8889-416e-9faa-903d99ee88b6","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:38.600Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"97fbb0b2-124b-4b99-ae71-53f4e7f487a6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:38.600Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"97fbb0b2-124b-4b99-ae71-53f4e7f487a6","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:38.600Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"97fbb0b2-124b-4b99-ae71-53f4e7f487a6","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:38.600Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"898b3977-cc0a-4572-990c-41ad24c0a8ca","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:38.600Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"97fbb0b2-124b-4b99-ae71-53f4e7f487a6"}
{"level":"info","ts":"2026-09-18T09:46:38.600Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"97fbb0b2-124b-4b99-ae71-53f4e7f487a6","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:38.600Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"97fbb0b2-124b-4b99-ae71-53f4e7f487a6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:38.600Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"97fbb0b2-124b-4b99-ae71-53f4e7f487a6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:38.600Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"898b3977-cc0a-4572-990c-41ad24c0a8ca","rayCluster":"stop-zllpq","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6b787086-138e-4763-8d41-1560b1c85e5a"}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6b787086-138e-4763-8d41-1560b1c85e5a"}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6b787086-138e-4763-8d41-1560b1c85e5a"}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6b787086-138e-4763-8d41-1560b1c85e5a","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6b787086-138e-4763-8d41-1560b1c85e5a","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6b787086-138e-4763-8d41-1560b1c85e5a","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6b787086-138e-4763-8d41-1560b1c85e5a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6b787086-138e-4763-8d41-1560b1c85e5a","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6b787086-138e-4763-8d41-1560b1c85e5a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6b787086-138e-4763-8d41-1560b1c85e5a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6b787086-138e-4763-8d41-1560b1c85e5a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6b787086-138e-4763-8d41-1560b1c85e5a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"b2f20678-7bc4-4786-b527-4ef9a4ce0077","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6b787086-138e-4763-8d41-1560b1c85e5a","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6b787086-138e-4763-8d41-1560b1c85e5a","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6b787086-138e-4763-8d41-1560b1c85e5a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:38.666Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"6b787086-138e-4763-8d41-1560b1c85e5a","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:38.667Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"da131e7f-825f-42dc-9332-c26a12ab7b6f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:38.667Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"da131e7f-825f-42dc-9332-c26a12ab7b6f","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:38.667Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"da131e7f-825f-42dc-9332-c26a12ab7b6f","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:38.667Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"da131e7f-825f-42dc-9332-c26a12ab7b6f"}
{"level":"info","ts":"2026-09-18T09:46:38.667Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"da131e7f-825f-42dc-9332-c26a12ab7b6f","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:38.667Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"da131e7f-825f-42dc-9332-c26a12ab7b6f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:38.671Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"e8f00739-8889-416e-9faa-903d99ee88b6","name":"stop-zllpq-head"}
{"level":"info","ts":"2026-09-18T09:46:38.684Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"e8f00739-8889-416e-9faa-903d99ee88b6","name":"stop-zllpq-workers"}
{"level":"info","ts":"2026-09-18T09:46:38.684Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"e8f00739-8889-416e-9faa-903d99ee88b6","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:38.874Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"ce7b1c99-7994-41eb-8743-27a24c1c18ad","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:38.874Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"ce7b1c99-7994-41eb-8743-27a24c1c18ad","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:38.874Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"ce7b1c99-7994-41eb-8743-27a24c1c18ad","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:38.875Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"ce7b1c99-7994-41eb-8743-27a24c1c18ad"}
{"level":"info","ts":"2026-09-18T09:46:38.875Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"ce7b1c99-7994-41eb-8743-27a24c1c18ad","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:38.875Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"ce7b1c99-7994-41eb-8743-27a24c1c18ad","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:39.124Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3827355c-6358-4b36-87b4-dbf02a2d2c9c"}
{"level":"info","ts":"2026-09-18T09:46:39.124Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3827355c-6358-4b36-87b4-dbf02a2d2c9c"}
{"level":"info","ts":"2026-09-18T09:46:39.124Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3827355c-6358-4b36-87b4-dbf02a2d2c9c"}
{"level":"info","ts":"2026-09-18T09:46:39.124Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3827355c-6358-4b36-87b4-dbf02a2d2c9c","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:39.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3827355c-6358-4b36-87b4-dbf02a2d2c9c","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:39.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3827355c-6358-4b36-87b4-dbf02a2d2c9c","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:39.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3827355c-6358-4b36-87b4-dbf02a2d2c9c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:39.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3827355c-6358-4b36-87b4-dbf02a2d2c9c","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:39.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3827355c-6358-4b36-87b4-dbf02a2d2c9c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:39.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3827355c-6358-4b36-87b4-dbf02a2d2c9c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:39.124Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3827355c-6358-4b36-87b4-dbf02a2d2c9c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:39.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3827355c-6358-4b36-87b4-dbf02a2d2c9c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:39.124Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3827355c-6358-4b36-87b4-dbf02a2d2c9c","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:39.124Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3827355c-6358-4b36-87b4-dbf02a2d2c9c","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:39.125Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3827355c-6358-4b36-87b4-dbf02a2d2c9c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:39.125Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3827355c-6358-4b36-87b4-dbf02a2d2c9c","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:39.145Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d8949b99-f66f-4f92-8945-e9e8e88b4f02"}
{"level":"info","ts":"2026-09-18T09:46:39.145Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d8949b99-f66f-4f92-8945-e9e8e88b4f02"}
{"level":"info","ts":"2026-09-18T09:46:39.145Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d8949b99-f66f-4f92-8945-e9e8e88b4f02"}
{"level":"info","ts":"2026-09-18T09:46:39.145Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d8949b99-f66f-4f92-8945-e9e8e88b4f02","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:39.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d8949b99-f66f-4f92-8945-e9e8e88b4f02","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:39.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d8949b99-f66f-4f92-8945-e9e8e88b4f02","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:39.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d8949b99-f66f-4f92-8945-e9e8e88b4f02","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:39.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d8949b99-f66f-4f92-8945-e9e8e88b4f02","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:39.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d8949b99-f66f-4f92-8945-e9e8e88b4f02","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:39.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d8949b99-f66f-4f92-8945-e9e8e88b4f02","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:39.145Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d8949b99-f66f-4f92-8945-e9e8e88b4f02","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:39.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d8949b99-f66f-4f92-8945-e9e8e88b4f02","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:39.145Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d8949b99-f66f-4f92-8945-e9e8e88b4f02","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:39.145Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d8949b99-f66f-4f92-8945-e9e8e88b4f02","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:39.145Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d8949b99-f66f-4f92-8945-e9e8e88b4f02","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:39.145Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"d8949b99-f66f-4f92-8945-e9e8e88b4f02","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:39.305Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"66442ed1-44f4-4515-9c7f-5d5dbcc11385"}
{"level":"info","ts":"2026-09-18T09:46:39.305Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"66442ed1-44f4-4515-9c7f-5d5dbcc11385"}
{"level":"info","ts":"2026-09-18T09:46:39.305Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"66442ed1-44f4-4515-9c7f-5d5dbcc11385"}
{"level":"info","ts":"2026-09-18T09:46:39.305Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"66442ed1-44f4-4515-9c7f-5d5dbcc11385","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:39.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"66442ed1-44f4-4515-9c7f-5d5dbcc11385","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:39.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"66442ed1-44f4-4515-9c7f-5d5dbcc11385","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:39.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"66442ed1-44f4-4515-9c7f-5d5dbcc11385","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:39.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"66442ed1-44f4-4515-9c7f-5d5dbcc11385","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:39.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"66442ed1-44f4-4515-9c7f-5d5dbcc11385","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:39.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"66442ed1-44f4-4515-9c7f-5d5dbcc11385","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:39.306Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"66442ed1-44f4-4515-9c7f-5d5dbcc11385","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:39.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"66442ed1-44f4-4515-9c7f-5d5dbcc11385","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:39.306Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"66442ed1-44f4-4515-9c7f-5d5dbcc11385","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:39.306Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"66442ed1-44f4-4515-9c7f-5d5dbcc11385","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:39.306Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"66442ed1-44f4-4515-9c7f-5d5dbcc11385","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:39.306Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"66442ed1-44f4-4515-9c7f-5d5dbcc11385","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:39.484Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c427aac9-99aa-4c11-ba80-e9112f969129"}
{"level":"info","ts":"2026-09-18T09:46:39.484Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c427aac9-99aa-4c11-ba80-e9112f969129"}
{"level":"info","ts":"2026-09-18T09:46:39.484Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c427aac9-99aa-4c11-ba80-e9112f969129"}
{"level":"info","ts":"2026-09-18T09:46:39.484Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c427aac9-99aa-4c11-ba80-e9112f969129","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:39.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c427aac9-99aa-4c11-ba80-e9112f969129","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:39.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c427aac9-99aa-4c11-ba80-e9112f969129","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:39.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c427aac9-99aa-4c11-ba80-e9112f969129","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:39.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c427aac9-99aa-4c11-ba80-e9112f969129","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:39.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c427aac9-99aa-4c11-ba80-e9112f969129","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:39.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c427aac9-99aa-4c11-ba80-e9112f969129","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:39.485Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c427aac9-99aa-4c11-ba80-e9112f969129","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:39.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c427aac9-99aa-4c11-ba80-e9112f969129","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:39.485Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c427aac9-99aa-4c11-ba80-e9112f969129","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:39.485Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c427aac9-99aa-4c11-ba80-e9112f969129","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:39.485Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"c427aac9-99aa-4c11-ba80-e9112f969129","name":"stop-zllpq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:39.497Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"4fd61bd8-ac75-434d-9b17-7845eef3b8f6","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:39.497Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"231f2d0b-bf96-46bc-9cce-227473332355","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:39.497Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"231f2d0b-bf96-46bc-9cce-227473332355","pod":"stop-zllpq-head-ctt42","ip":"10.132.0.32"}
{"level":"info","ts":"2026-09-18T09:46:39.497Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"231f2d0b-bf96-46bc-9cce-227473332355","pod":"stop-zllpq-small-group-worker-bfxtm","ip":"10.133.0.30"}
{"level":"info","ts":"2026-09-18T09:46:39.497Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d8160a1a-9863-4ab9-9b10-a24b55bdd099","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:39.497Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d8160a1a-9863-4ab9-9b10-a24b55bdd099","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:39.497Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d8160a1a-9863-4ab9-9b10-a24b55bdd099","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:39.498Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d8160a1a-9863-4ab9-9b10-a24b55bdd099"}
{"level":"info","ts":"2026-09-18T09:46:39.498Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d8160a1a-9863-4ab9-9b10-a24b55bdd099","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:39.498Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d8160a1a-9863-4ab9-9b10-a24b55bdd099","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:39.498Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d8160a1a-9863-4ab9-9b10-a24b55bdd099","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:39.510Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"3dc2d0d6-03e1-42f9-8304-97bc48e4a02b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:39.510Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"3dc2d0d6-03e1-42f9-8304-97bc48e4a02b","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:39.510Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"3dc2d0d6-03e1-42f9-8304-97bc48e4a02b","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:39.510Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"3dc2d0d6-03e1-42f9-8304-97bc48e4a02b"}
{"level":"info","ts":"2026-09-18T09:46:39.510Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"3dc2d0d6-03e1-42f9-8304-97bc48e4a02b","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:39.510Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"3dc2d0d6-03e1-42f9-8304-97bc48e4a02b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:39.511Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"231f2d0b-bf96-46bc-9cce-227473332355","certificate":"ray-head-cert-stop-zllpq","dnsNames":["stop-zllpq-head-svc","localhost","stop-zllpq-head-svc.test-ns-q2lj2.svc","stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local"],"ipAddresses":["10.132.0.32","10.133.0.30","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:46:39.512Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"4fd61bd8-ac75-434d-9b17-7845eef3b8f6","name":"stop-zllpq-head"}
{"level":"info","ts":"2026-09-18T09:46:39.523Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"231f2d0b-bf96-46bc-9cce-227473332355","certificate":"ray-worker-cert-stop-zllpq","dnsNames":["stop-zllpq-worker-svc","localhost","stop-zllpq-small-group","stop-zllpq-worker-svc.test-ns-q2lj2.svc","stop-zllpq-worker-svc.test-ns-q2lj2.svc.cluster.local","*.stop-zllpq-worker-svc.test-ns-q2lj2.svc","*.stop-zllpq-worker-svc.test-ns-q2lj2.svc.cluster.local","*-worker-*.test-ns-q2lj2.svc","*-worker-*.test-ns-q2lj2.svc.cluster.local"],"ipAddresses":["10.132.0.32","10.133.0.30","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:46:39.524Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"231f2d0b-bf96-46bc-9cce-227473332355","rayCluster":"stop-zllpq","podIPs":["10.132.0.32","10.133.0.30"]}
{"level":"info","ts":"2026-09-18T09:46:39.529Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"4fd61bd8-ac75-434d-9b17-7845eef3b8f6","name":"stop-zllpq-workers"}
{"level":"info","ts":"2026-09-18T09:46:39.529Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"4fd61bd8-ac75-434d-9b17-7845eef3b8f6","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:40.601Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3b13b12e-25a5-472b-b9ec-2e454da6113b"}
{"level":"info","ts":"2026-09-18T09:46:40.601Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3b13b12e-25a5-472b-b9ec-2e454da6113b"}
{"level":"info","ts":"2026-09-18T09:46:40.601Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3b13b12e-25a5-472b-b9ec-2e454da6113b"}
{"level":"info","ts":"2026-09-18T09:46:40.601Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3b13b12e-25a5-472b-b9ec-2e454da6113b","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:40.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3b13b12e-25a5-472b-b9ec-2e454da6113b","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:40.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3b13b12e-25a5-472b-b9ec-2e454da6113b","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:40.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3b13b12e-25a5-472b-b9ec-2e454da6113b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:40.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3b13b12e-25a5-472b-b9ec-2e454da6113b","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:40.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3b13b12e-25a5-472b-b9ec-2e454da6113b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:40.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3b13b12e-25a5-472b-b9ec-2e454da6113b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:40.601Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3b13b12e-25a5-472b-b9ec-2e454da6113b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:40.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3b13b12e-25a5-472b-b9ec-2e454da6113b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:40.601Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3b13b12e-25a5-472b-b9ec-2e454da6113b","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:40.601Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3b13b12e-25a5-472b-b9ec-2e454da6113b","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:40.601Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3b13b12e-25a5-472b-b9ec-2e454da6113b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:40.601Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"3b13b12e-25a5-472b-b9ec-2e454da6113b","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:41.875Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"f85c28db-1dc5-4f74-92ca-7a3b60010115","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:41.875Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"f85c28db-1dc5-4f74-92ca-7a3b60010115","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:41.875Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"f85c28db-1dc5-4f74-92ca-7a3b60010115","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:41.875Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"f85c28db-1dc5-4f74-92ca-7a3b60010115"}
{"level":"info","ts":"2026-09-18T09:46:41.875Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"f85c28db-1dc5-4f74-92ca-7a3b60010115","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:41.875Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"f85c28db-1dc5-4f74-92ca-7a3b60010115","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:44.332Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"bb3f7e11-5431-4b4a-8fcb-7242259de27a"}
{"level":"info","ts":"2026-09-18T09:46:44.332Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"bb3f7e11-5431-4b4a-8fcb-7242259de27a"}
{"level":"info","ts":"2026-09-18T09:46:44.332Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"bb3f7e11-5431-4b4a-8fcb-7242259de27a"}
{"level":"info","ts":"2026-09-18T09:46:44.333Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"bb3f7e11-5431-4b4a-8fcb-7242259de27a","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:44.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"bb3f7e11-5431-4b4a-8fcb-7242259de27a","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:44.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"bb3f7e11-5431-4b4a-8fcb-7242259de27a","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:44.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"bb3f7e11-5431-4b4a-8fcb-7242259de27a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:44.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"bb3f7e11-5431-4b4a-8fcb-7242259de27a","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:44.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"bb3f7e11-5431-4b4a-8fcb-7242259de27a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:44.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"bb3f7e11-5431-4b4a-8fcb-7242259de27a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:44.333Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"bb3f7e11-5431-4b4a-8fcb-7242259de27a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:44.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"bb3f7e11-5431-4b4a-8fcb-7242259de27a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:44.333Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"bb3f7e11-5431-4b4a-8fcb-7242259de27a","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:44.333Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"bb3f7e11-5431-4b4a-8fcb-7242259de27a","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:44.333Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"bb3f7e11-5431-4b4a-8fcb-7242259de27a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:44.333Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"bb3f7e11-5431-4b4a-8fcb-7242259de27a","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:44.510Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"706edea7-a036-4478-adbc-0bcfe5ec373a"}
{"level":"info","ts":"2026-09-18T09:46:44.510Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"706edea7-a036-4478-adbc-0bcfe5ec373a"}
{"level":"info","ts":"2026-09-18T09:46:44.510Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"706edea7-a036-4478-adbc-0bcfe5ec373a"}
{"level":"info","ts":"2026-09-18T09:46:44.510Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"706edea7-a036-4478-adbc-0bcfe5ec373a","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:44.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"706edea7-a036-4478-adbc-0bcfe5ec373a","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:44.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"706edea7-a036-4478-adbc-0bcfe5ec373a","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:44.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"706edea7-a036-4478-adbc-0bcfe5ec373a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:44.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"706edea7-a036-4478-adbc-0bcfe5ec373a","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:44.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"706edea7-a036-4478-adbc-0bcfe5ec373a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:44.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"706edea7-a036-4478-adbc-0bcfe5ec373a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:44.510Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"706edea7-a036-4478-adbc-0bcfe5ec373a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:44.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"706edea7-a036-4478-adbc-0bcfe5ec373a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:44.510Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"706edea7-a036-4478-adbc-0bcfe5ec373a","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:44.510Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"706edea7-a036-4478-adbc-0bcfe5ec373a","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:44.510Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"706edea7-a036-4478-adbc-0bcfe5ec373a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:44.510Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"706edea7-a036-4478-adbc-0bcfe5ec373a","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:44.876Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d06fc860-d632-4d74-9597-e0c65f6a48e0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:44.876Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d06fc860-d632-4d74-9597-e0c65f6a48e0","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:44.876Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d06fc860-d632-4d74-9597-e0c65f6a48e0","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:44.876Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d06fc860-d632-4d74-9597-e0c65f6a48e0"}
{"level":"info","ts":"2026-09-18T09:46:44.876Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d06fc860-d632-4d74-9597-e0c65f6a48e0","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:44.876Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"d06fc860-d632-4d74-9597-e0c65f6a48e0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:45.336Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"5eef4714-a972-4bcf-b3fd-4d8e476319e6"}
{"level":"info","ts":"2026-09-18T09:46:45.336Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"5eef4714-a972-4bcf-b3fd-4d8e476319e6"}
{"level":"info","ts":"2026-09-18T09:46:45.336Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"5eef4714-a972-4bcf-b3fd-4d8e476319e6"}
{"level":"info","ts":"2026-09-18T09:46:45.336Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"5eef4714-a972-4bcf-b3fd-4d8e476319e6","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:45.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"5eef4714-a972-4bcf-b3fd-4d8e476319e6","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:45.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"5eef4714-a972-4bcf-b3fd-4d8e476319e6","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:45.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"5eef4714-a972-4bcf-b3fd-4d8e476319e6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:45.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"5eef4714-a972-4bcf-b3fd-4d8e476319e6","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:45.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"5eef4714-a972-4bcf-b3fd-4d8e476319e6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:45.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"5eef4714-a972-4bcf-b3fd-4d8e476319e6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:45.336Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"5eef4714-a972-4bcf-b3fd-4d8e476319e6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:45.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"5eef4714-a972-4bcf-b3fd-4d8e476319e6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:45.337Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"5eef4714-a972-4bcf-b3fd-4d8e476319e6","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:45.337Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"5eef4714-a972-4bcf-b3fd-4d8e476319e6","podName":"stop-zllpq-head-ctt42","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:46:45.337Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"5eef4714-a972-4bcf-b3fd-4d8e476319e6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:45.337Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"5eef4714-a972-4bcf-b3fd-4d8e476319e6","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:45.524Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"1a8716af-1ea5-4e21-9d40-83700db94468"}
{"level":"info","ts":"2026-09-18T09:46:45.524Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"1a8716af-1ea5-4e21-9d40-83700db94468"}
{"level":"info","ts":"2026-09-18T09:46:45.525Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"1a8716af-1ea5-4e21-9d40-83700db94468"}
{"level":"info","ts":"2026-09-18T09:46:45.525Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"1a8716af-1ea5-4e21-9d40-83700db94468","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:45.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"1a8716af-1ea5-4e21-9d40-83700db94468","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:45.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"1a8716af-1ea5-4e21-9d40-83700db94468","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:45.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"1a8716af-1ea5-4e21-9d40-83700db94468","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:45.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"1a8716af-1ea5-4e21-9d40-83700db94468","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:45.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"1a8716af-1ea5-4e21-9d40-83700db94468","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:45.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"1a8716af-1ea5-4e21-9d40-83700db94468","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:45.525Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"1a8716af-1ea5-4e21-9d40-83700db94468","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:45.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"1a8716af-1ea5-4e21-9d40-83700db94468","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:45.525Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"1a8716af-1ea5-4e21-9d40-83700db94468","podName":"stop-zllpq-head-ctt42","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:46:45.525Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"1a8716af-1ea5-4e21-9d40-83700db94468","podName":"stop-zllpq-head-ctt42","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:46:45.525Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"1a8716af-1ea5-4e21-9d40-83700db94468","name":"stop-zllpq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:45.541Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"3b43b316-0b41-4535-9ce7-4cd19854cf3f","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:45.541Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"70d3fae9-ea19-48f3-a7d5-7f1bf36fe15d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:45.542Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"70d3fae9-ea19-48f3-a7d5-7f1bf36fe15d","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:45.542Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"70d3fae9-ea19-48f3-a7d5-7f1bf36fe15d","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:45.542Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"3b43b316-0b41-4535-9ce7-4cd19854cf3f","pod":"stop-zllpq-head-ctt42","ip":"10.132.0.32"}
{"level":"info","ts":"2026-09-18T09:46:45.542Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"3b43b316-0b41-4535-9ce7-4cd19854cf3f","pod":"stop-zllpq-small-group-worker-bfxtm","ip":"10.133.0.30"}
{"level":"info","ts":"2026-09-18T09:46:45.542Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"3b43b316-0b41-4535-9ce7-4cd19854cf3f","rayCluster":"stop-zllpq","podIPs":["10.132.0.32","10.133.0.30"]}
{"level":"info","ts":"2026-09-18T09:46:45.542Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"70d3fae9-ea19-48f3-a7d5-7f1bf36fe15d"}
{"level":"info","ts":"2026-09-18T09:46:45.542Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"70d3fae9-ea19-48f3-a7d5-7f1bf36fe15d","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:45.542Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"bc596758-02ff-4d75-a64d-e40e68ae2eef","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:45.542Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"70d3fae9-ea19-48f3-a7d5-7f1bf36fe15d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:45.542Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"70d3fae9-ea19-48f3-a7d5-7f1bf36fe15d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:45.567Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"4d4b941c-14a0-4da9-830f-02017725b793","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:45.567Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"4d4b941c-14a0-4da9-830f-02017725b793","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:45.567Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"4d4b941c-14a0-4da9-830f-02017725b793","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:45.567Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"4d4b941c-14a0-4da9-830f-02017725b793"}
{"level":"info","ts":"2026-09-18T09:46:45.567Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"4d4b941c-14a0-4da9-830f-02017725b793","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:45.567Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"4d4b941c-14a0-4da9-830f-02017725b793","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:45.572Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"bc596758-02ff-4d75-a64d-e40e68ae2eef","name":"stop-zllpq-head"}
{"level":"info","ts":"2026-09-18T09:46:45.589Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"bc596758-02ff-4d75-a64d-e40e68ae2eef","name":"stop-zllpq-workers"}
{"level":"info","ts":"2026-09-18T09:46:45.590Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"bc596758-02ff-4d75-a64d-e40e68ae2eef","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:47.544Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"2548abe1-71a4-4450-828c-0c82e5031444"}
{"level":"info","ts":"2026-09-18T09:46:47.544Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"2548abe1-71a4-4450-828c-0c82e5031444"}
{"level":"info","ts":"2026-09-18T09:46:47.544Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"2548abe1-71a4-4450-828c-0c82e5031444"}
{"level":"info","ts":"2026-09-18T09:46:47.544Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"2548abe1-71a4-4450-828c-0c82e5031444","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:47.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"2548abe1-71a4-4450-828c-0c82e5031444","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:47.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"2548abe1-71a4-4450-828c-0c82e5031444","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:47.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"2548abe1-71a4-4450-828c-0c82e5031444","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:47.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"2548abe1-71a4-4450-828c-0c82e5031444","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:47.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"2548abe1-71a4-4450-828c-0c82e5031444","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:47.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"2548abe1-71a4-4450-828c-0c82e5031444","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:47.544Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"2548abe1-71a4-4450-828c-0c82e5031444","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:47.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"2548abe1-71a4-4450-828c-0c82e5031444","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:47.544Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"2548abe1-71a4-4450-828c-0c82e5031444","podName":"stop-zllpq-head-ctt42","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:46:47.544Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"2548abe1-71a4-4450-828c-0c82e5031444","podName":"stop-zllpq-head-ctt42","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:46:47.544Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"2548abe1-71a4-4450-828c-0c82e5031444","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:47.544Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"2548abe1-71a4-4450-828c-0c82e5031444","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:47.877Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"71b40f33-0f14-44f1-9c50-24039ab36e85","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:47.877Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"71b40f33-0f14-44f1-9c50-24039ab36e85","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:47.877Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"71b40f33-0f14-44f1-9c50-24039ab36e85","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:47.877Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"71b40f33-0f14-44f1-9c50-24039ab36e85"}
{"level":"info","ts":"2026-09-18T09:46:47.877Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"71b40f33-0f14-44f1-9c50-24039ab36e85","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:47.877Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"71b40f33-0f14-44f1-9c50-24039ab36e85","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:50.878Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"09809f4d-d199-4657-95c4-7cc1412e80d9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:50.878Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"09809f4d-d199-4657-95c4-7cc1412e80d9","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:50.878Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"09809f4d-d199-4657-95c4-7cc1412e80d9","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:50.878Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"09809f4d-d199-4657-95c4-7cc1412e80d9"}
{"level":"info","ts":"2026-09-18T09:46:50.878Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"09809f4d-d199-4657-95c4-7cc1412e80d9","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:50.878Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"09809f4d-d199-4657-95c4-7cc1412e80d9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:53.879Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"2a24d2e8-08f1-4ae8-bf8a-975756a1d301","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:53.879Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"2a24d2e8-08f1-4ae8-bf8a-975756a1d301","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:53.879Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"2a24d2e8-08f1-4ae8-bf8a-975756a1d301","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:53.879Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"2a24d2e8-08f1-4ae8-bf8a-975756a1d301"}
{"level":"info","ts":"2026-09-18T09:46:53.879Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"2a24d2e8-08f1-4ae8-bf8a-975756a1d301","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:53.879Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"2a24d2e8-08f1-4ae8-bf8a-975756a1d301","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:54.391Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"fad11ae3-75d5-4129-a3ab-f415d87aff12"}
{"level":"info","ts":"2026-09-18T09:46:54.391Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"fad11ae3-75d5-4129-a3ab-f415d87aff12"}
{"level":"info","ts":"2026-09-18T09:46:54.391Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"fad11ae3-75d5-4129-a3ab-f415d87aff12"}
{"level":"info","ts":"2026-09-18T09:46:54.391Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"fad11ae3-75d5-4129-a3ab-f415d87aff12","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:54.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"fad11ae3-75d5-4129-a3ab-f415d87aff12","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:54.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"fad11ae3-75d5-4129-a3ab-f415d87aff12","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:54.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"fad11ae3-75d5-4129-a3ab-f415d87aff12","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:54.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"fad11ae3-75d5-4129-a3ab-f415d87aff12","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:54.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"fad11ae3-75d5-4129-a3ab-f415d87aff12","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:54.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"fad11ae3-75d5-4129-a3ab-f415d87aff12","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:54.391Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"fad11ae3-75d5-4129-a3ab-f415d87aff12","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:54.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"fad11ae3-75d5-4129-a3ab-f415d87aff12","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:54.391Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"fad11ae3-75d5-4129-a3ab-f415d87aff12","podName":"stop-zllpq-head-ctt42","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:46:54.392Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"fad11ae3-75d5-4129-a3ab-f415d87aff12","podName":"stop-zllpq-head-ctt42","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:46:54.392Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"fad11ae3-75d5-4129-a3ab-f415d87aff12","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:54.392Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"fad11ae3-75d5-4129-a3ab-f415d87aff12","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:55.402Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"69e68798-354b-4d92-9397-939cbedda5a6"}
{"level":"info","ts":"2026-09-18T09:46:55.402Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"69e68798-354b-4d92-9397-939cbedda5a6"}
{"level":"info","ts":"2026-09-18T09:46:55.402Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"69e68798-354b-4d92-9397-939cbedda5a6"}
{"level":"info","ts":"2026-09-18T09:46:55.402Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"69e68798-354b-4d92-9397-939cbedda5a6","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:55.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"69e68798-354b-4d92-9397-939cbedda5a6","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:55.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"69e68798-354b-4d92-9397-939cbedda5a6","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:55.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"69e68798-354b-4d92-9397-939cbedda5a6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:55.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"69e68798-354b-4d92-9397-939cbedda5a6","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:55.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"69e68798-354b-4d92-9397-939cbedda5a6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:55.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"69e68798-354b-4d92-9397-939cbedda5a6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:55.403Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"69e68798-354b-4d92-9397-939cbedda5a6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:55.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"69e68798-354b-4d92-9397-939cbedda5a6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:55.403Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"69e68798-354b-4d92-9397-939cbedda5a6","podName":"stop-zllpq-head-ctt42","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:46:55.403Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"69e68798-354b-4d92-9397-939cbedda5a6","podName":"stop-zllpq-head-ctt42","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:46:55.403Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"69e68798-354b-4d92-9397-939cbedda5a6","name":"stop-zllpq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:55.416Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"366ac9f3-eb01-4f43-94dd-5448cdd8179e","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:55.416Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"47684213-6072-48af-bdbe-7f30cdcd5561","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:55.416Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"47684213-6072-48af-bdbe-7f30cdcd5561","pod":"stop-zllpq-small-group-worker-bfxtm","ip":"10.133.0.30"}
{"level":"info","ts":"2026-09-18T09:46:55.416Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"47684213-6072-48af-bdbe-7f30cdcd5561","pod":"stop-zllpq-head-ctt42","ip":"10.132.0.32"}
{"level":"info","ts":"2026-09-18T09:46:55.416Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"47684213-6072-48af-bdbe-7f30cdcd5561","rayCluster":"stop-zllpq","podIPs":["10.133.0.30","10.132.0.32"]}
{"level":"info","ts":"2026-09-18T09:46:55.416Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"bb01616f-64cd-407a-833e-459d2f4e9b95","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:55.416Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"bb01616f-64cd-407a-833e-459d2f4e9b95","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:55.416Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"bb01616f-64cd-407a-833e-459d2f4e9b95","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:55.416Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"bb01616f-64cd-407a-833e-459d2f4e9b95"}
{"level":"info","ts":"2026-09-18T09:46:55.416Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"bb01616f-64cd-407a-833e-459d2f4e9b95","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:55.416Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"bb01616f-64cd-407a-833e-459d2f4e9b95","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:55.416Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"bb01616f-64cd-407a-833e-459d2f4e9b95","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:55.429Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"366ac9f3-eb01-4f43-94dd-5448cdd8179e","name":"stop-zllpq-head"}
{"level":"info","ts":"2026-09-18T09:46:55.429Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"3cc3c8b5-c47f-4793-9da3-b14023f173ca","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:55.429Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"3cc3c8b5-c47f-4793-9da3-b14023f173ca","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:55.429Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"3cc3c8b5-c47f-4793-9da3-b14023f173ca","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:55.429Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"3cc3c8b5-c47f-4793-9da3-b14023f173ca"}
{"level":"info","ts":"2026-09-18T09:46:55.429Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"3cc3c8b5-c47f-4793-9da3-b14023f173ca","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:55.429Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"3cc3c8b5-c47f-4793-9da3-b14023f173ca","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:55.442Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"366ac9f3-eb01-4f43-94dd-5448cdd8179e","name":"stop-zllpq-workers"}
{"level":"info","ts":"2026-09-18T09:46:55.442Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"366ac9f3-eb01-4f43-94dd-5448cdd8179e","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:56.550Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"0f75c15f-8b38-4721-ae8f-039ac527718f"}
{"level":"info","ts":"2026-09-18T09:46:56.550Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"0f75c15f-8b38-4721-ae8f-039ac527718f"}
{"level":"info","ts":"2026-09-18T09:46:56.550Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"0f75c15f-8b38-4721-ae8f-039ac527718f"}
{"level":"info","ts":"2026-09-18T09:46:56.550Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"0f75c15f-8b38-4721-ae8f-039ac527718f","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:56.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"0f75c15f-8b38-4721-ae8f-039ac527718f","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:56.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"0f75c15f-8b38-4721-ae8f-039ac527718f","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:56.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"0f75c15f-8b38-4721-ae8f-039ac527718f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:56.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"0f75c15f-8b38-4721-ae8f-039ac527718f","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:56.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"0f75c15f-8b38-4721-ae8f-039ac527718f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:56.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"0f75c15f-8b38-4721-ae8f-039ac527718f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:56.550Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"0f75c15f-8b38-4721-ae8f-039ac527718f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:56.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"0f75c15f-8b38-4721-ae8f-039ac527718f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:56.550Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"0f75c15f-8b38-4721-ae8f-039ac527718f","podName":"stop-zllpq-small-group-worker-bfxtm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:46:56.550Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"0f75c15f-8b38-4721-ae8f-039ac527718f","podName":"stop-zllpq-small-group-worker-bfxtm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:46:56.550Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"0f75c15f-8b38-4721-ae8f-039ac527718f","name":"stop-zllpq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:46:56.564Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f1748ba6-3631-459d-9ef8-795d2f02ad32","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:56.564Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"66d74782-fe0e-45f8-8de5-614b5c314abc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:56.564Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"66d74782-fe0e-45f8-8de5-614b5c314abc","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:56.564Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"66d74782-fe0e-45f8-8de5-614b5c314abc","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:56.564Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f1748ba6-3631-459d-9ef8-795d2f02ad32","pod":"stop-zllpq-head-ctt42","ip":"10.132.0.32"}
{"level":"info","ts":"2026-09-18T09:46:56.564Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f1748ba6-3631-459d-9ef8-795d2f02ad32","pod":"stop-zllpq-small-group-worker-bfxtm","ip":"10.133.0.30"}
{"level":"info","ts":"2026-09-18T09:46:56.564Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f1748ba6-3631-459d-9ef8-795d2f02ad32","rayCluster":"stop-zllpq","podIPs":["10.132.0.32","10.133.0.30"]}
{"level":"info","ts":"2026-09-18T09:46:56.564Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"66d74782-fe0e-45f8-8de5-614b5c314abc"}
{"level":"info","ts":"2026-09-18T09:46:56.564Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"66d74782-fe0e-45f8-8de5-614b5c314abc","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:56.564Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"66d74782-fe0e-45f8-8de5-614b5c314abc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:56.564Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"66d74782-fe0e-45f8-8de5-614b5c314abc","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:46:56.564Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"3e80ab17-086e-4ac1-8d02-50efeaddf584","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:56.580Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"276fa37a-4ab3-4032-9162-7ca10ad2d510","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:56.580Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"276fa37a-4ab3-4032-9162-7ca10ad2d510","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:56.580Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"276fa37a-4ab3-4032-9162-7ca10ad2d510","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:56.580Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"276fa37a-4ab3-4032-9162-7ca10ad2d510"}
{"level":"info","ts":"2026-09-18T09:46:56.580Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"276fa37a-4ab3-4032-9162-7ca10ad2d510","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:56.580Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"276fa37a-4ab3-4032-9162-7ca10ad2d510","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:56.582Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"3e80ab17-086e-4ac1-8d02-50efeaddf584","name":"stop-zllpq-head"}
{"level":"info","ts":"2026-09-18T09:46:56.594Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"3e80ab17-086e-4ac1-8d02-50efeaddf584","name":"stop-zllpq-workers"}
{"level":"info","ts":"2026-09-18T09:46:56.594Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"3e80ab17-086e-4ac1-8d02-50efeaddf584","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:56.880Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"225f88a1-e887-40c3-b360-450dcf4cf820","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:56.880Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"225f88a1-e887-40c3-b360-450dcf4cf820","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:56.880Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"225f88a1-e887-40c3-b360-450dcf4cf820","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:56.880Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"225f88a1-e887-40c3-b360-450dcf4cf820"}
{"level":"info","ts":"2026-09-18T09:46:56.880Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"225f88a1-e887-40c3-b360-450dcf4cf820","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:56.880Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"225f88a1-e887-40c3-b360-450dcf4cf820","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:46:57.416Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"20932640-5629-4c28-88a0-44bf69dc0627"}
{"level":"info","ts":"2026-09-18T09:46:57.416Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"20932640-5629-4c28-88a0-44bf69dc0627"}
{"level":"info","ts":"2026-09-18T09:46:57.416Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"20932640-5629-4c28-88a0-44bf69dc0627"}
{"level":"info","ts":"2026-09-18T09:46:57.416Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"20932640-5629-4c28-88a0-44bf69dc0627","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:46:57.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"20932640-5629-4c28-88a0-44bf69dc0627","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:46:57.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"20932640-5629-4c28-88a0-44bf69dc0627","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:57.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"20932640-5629-4c28-88a0-44bf69dc0627","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:46:57.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"20932640-5629-4c28-88a0-44bf69dc0627","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:46:57.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"20932640-5629-4c28-88a0-44bf69dc0627","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:57.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"20932640-5629-4c28-88a0-44bf69dc0627","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:46:57.417Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"20932640-5629-4c28-88a0-44bf69dc0627","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:46:57.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"20932640-5629-4c28-88a0-44bf69dc0627","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:46:57.417Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"20932640-5629-4c28-88a0-44bf69dc0627","podName":"stop-zllpq-small-group-worker-bfxtm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:46:57.417Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"20932640-5629-4c28-88a0-44bf69dc0627","podName":"stop-zllpq-small-group-worker-bfxtm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:46:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:46:57.417Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"20932640-5629-4c28-88a0-44bf69dc0627","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:46:57.417Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"20932640-5629-4c28-88a0-44bf69dc0627","seconds":300}
{"level":"info","ts":"2026-09-18T09:46:59.881Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"696e78cb-6a29-46a0-a512-e3f30e3a7e17","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:46:59.881Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"696e78cb-6a29-46a0-a512-e3f30e3a7e17","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:46:59.881Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"696e78cb-6a29-46a0-a512-e3f30e3a7e17","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:46:59.881Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"696e78cb-6a29-46a0-a512-e3f30e3a7e17"}
{"level":"info","ts":"2026-09-18T09:46:59.881Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"696e78cb-6a29-46a0-a512-e3f30e3a7e17","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:46:59.881Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"696e78cb-6a29-46a0-a512-e3f30e3a7e17","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:02.881Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"45ac427b-3eeb-45fb-a39a-a58f3802d6cb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:02.881Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"45ac427b-3eeb-45fb-a39a-a58f3802d6cb","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:02.881Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"45ac427b-3eeb-45fb-a39a-a58f3802d6cb","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:02.882Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"45ac427b-3eeb-45fb-a39a-a58f3802d6cb"}
{"level":"info","ts":"2026-09-18T09:47:02.882Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"45ac427b-3eeb-45fb-a39a-a58f3802d6cb","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:47:02.882Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"45ac427b-3eeb-45fb-a39a-a58f3802d6cb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:05.882Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"0c924dd1-e7f8-4be1-9809-966b744e0454","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:05.883Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"0c924dd1-e7f8-4be1-9809-966b744e0454","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:05.883Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"0c924dd1-e7f8-4be1-9809-966b744e0454","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:05.883Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"0c924dd1-e7f8-4be1-9809-966b744e0454"}
{"level":"info","ts":"2026-09-18T09:47:05.883Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"0c924dd1-e7f8-4be1-9809-966b744e0454","RayCluster":"stop-zllpq","State":""}
{"level":"info","ts":"2026-09-18T09:47:05.883Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"0c924dd1-e7f8-4be1-9809-966b744e0454","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:06.095Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"0aa736bd-81cd-4431-8195-99fc52672b30","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:06.095Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"0aa736bd-81cd-4431-8195-99fc52672b30","pod":"stop-zllpq-head-ctt42","ip":"10.132.0.32"}
{"level":"info","ts":"2026-09-18T09:47:06.095Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"0aa736bd-81cd-4431-8195-99fc52672b30","pod":"stop-zllpq-small-group-worker-bfxtm","ip":"10.133.0.30"}
{"level":"info","ts":"2026-09-18T09:47:06.095Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"0aa736bd-81cd-4431-8195-99fc52672b30","rayCluster":"stop-zllpq","podIPs":["10.132.0.32","10.133.0.30"]}
{"level":"info","ts":"2026-09-18T09:47:06.422Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"4652f47b-63f4-488b-ac3c-6b40c51c371c"}
{"level":"info","ts":"2026-09-18T09:47:06.422Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"4652f47b-63f4-488b-ac3c-6b40c51c371c"}
{"level":"info","ts":"2026-09-18T09:47:06.422Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"4652f47b-63f4-488b-ac3c-6b40c51c371c"}
{"level":"info","ts":"2026-09-18T09:47:06.422Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"4652f47b-63f4-488b-ac3c-6b40c51c371c","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:06.422Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"4652f47b-63f4-488b-ac3c-6b40c51c371c","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:47:06.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"4652f47b-63f4-488b-ac3c-6b40c51c371c","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:06.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"4652f47b-63f4-488b-ac3c-6b40c51c371c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:47:06.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"4652f47b-63f4-488b-ac3c-6b40c51c371c","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:06.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"4652f47b-63f4-488b-ac3c-6b40c51c371c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:06.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"4652f47b-63f4-488b-ac3c-6b40c51c371c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:47:06.423Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"4652f47b-63f4-488b-ac3c-6b40c51c371c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:47:06.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"4652f47b-63f4-488b-ac3c-6b40c51c371c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:06.423Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"4652f47b-63f4-488b-ac3c-6b40c51c371c","name":"stop-zllpq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:47:06.436Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"2647e9de-f4eb-4b7c-9c25-50634be554a8","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:06.436Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"8a59da58-b58f-4d72-a737-da385cfdeba6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:06.436Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"8a59da58-b58f-4d72-a737-da385cfdeba6","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:06.436Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"8a59da58-b58f-4d72-a737-da385cfdeba6","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:06.436Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"2647e9de-f4eb-4b7c-9c25-50634be554a8","pod":"stop-zllpq-small-group-worker-bfxtm","ip":"10.133.0.30"}
{"level":"info","ts":"2026-09-18T09:47:06.436Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"2647e9de-f4eb-4b7c-9c25-50634be554a8","pod":"stop-zllpq-head-ctt42","ip":"10.132.0.32"}
{"level":"info","ts":"2026-09-18T09:47:06.436Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"2647e9de-f4eb-4b7c-9c25-50634be554a8","rayCluster":"stop-zllpq","podIPs":["10.133.0.30","10.132.0.32"]}
{"level":"info","ts":"2026-09-18T09:47:06.436Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"8a59da58-b58f-4d72-a737-da385cfdeba6"}
{"level":"info","ts":"2026-09-18T09:47:06.436Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f0bdcc6c-9cfc-4d8a-806c-94ca2b84e290","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:06.436Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"8a59da58-b58f-4d72-a737-da385cfdeba6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:06.436Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"8a59da58-b58f-4d72-a737-da385cfdeba6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:47:06.449Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"c635609f-a7d3-4680-8f02-3cf75876a975","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:06.449Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"c635609f-a7d3-4680-8f02-3cf75876a975","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:06.449Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"c635609f-a7d3-4680-8f02-3cf75876a975","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:06.449Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"c635609f-a7d3-4680-8f02-3cf75876a975"}
{"level":"info","ts":"2026-09-18T09:47:06.449Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"c635609f-a7d3-4680-8f02-3cf75876a975","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:47:06.454Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f0bdcc6c-9cfc-4d8a-806c-94ca2b84e290","name":"stop-zllpq-head"}
{"level":"info","ts":"2026-09-18T09:47:06.463Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"c635609f-a7d3-4680-8f02-3cf75876a975","JobId":"stop-nmzdw"}
{"level":"info","ts":"2026-09-18T09:47:06.468Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f0bdcc6c-9cfc-4d8a-806c-94ca2b84e290","name":"stop-zllpq-workers"}
{"level":"info","ts":"2026-09-18T09:47:06.468Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"f0bdcc6c-9cfc-4d8a-806c-94ca2b84e290","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:07.325Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"f69cb7d4-e8a5-45a7-b8a4-24c6ee6ec335","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:07.325Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"f69cb7d4-e8a5-45a7-b8a4-24c6ee6ec335","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:07.325Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"f69cb7d4-e8a5-45a7-b8a4-24c6ee6ec335","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:07.325Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"f69cb7d4-e8a5-45a7-b8a4-24c6ee6ec335"}
{"level":"info","ts":"2026-09-18T09:47:07.325Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"f69cb7d4-e8a5-45a7-b8a4-24c6ee6ec335","rayCluster":"stop-zllpq","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:47:07.328Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"f69cb7d4-e8a5-45a7-b8a4-24c6ee6ec335","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:07.329Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"f69cb7d4-e8a5-45a7-b8a4-24c6ee6ec335","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:47:07.343Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"71497775-08b5-417f-986d-54800fa3daa2","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:07.343Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"71497775-08b5-417f-986d-54800fa3daa2","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:07.343Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"71497775-08b5-417f-986d-54800fa3daa2","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:07.344Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"71497775-08b5-417f-986d-54800fa3daa2"}
{"level":"info","ts":"2026-09-18T09:47:07.344Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"71497775-08b5-417f-986d-54800fa3daa2","rayCluster":"stop-zllpq","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:47:07.346Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"71497775-08b5-417f-986d-54800fa3daa2","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:08.436Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f0b21a0f-a4a5-4c35-90cc-dbcc8206da45"}
{"level":"info","ts":"2026-09-18T09:47:08.437Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f0b21a0f-a4a5-4c35-90cc-dbcc8206da45"}
{"level":"info","ts":"2026-09-18T09:47:08.437Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f0b21a0f-a4a5-4c35-90cc-dbcc8206da45"}
{"level":"info","ts":"2026-09-18T09:47:08.437Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f0b21a0f-a4a5-4c35-90cc-dbcc8206da45","1 head service found":"stop-zllpq-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:08.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f0b21a0f-a4a5-4c35-90cc-dbcc8206da45","Found 1 head Pod":"stop-zllpq-head-ctt42","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:47:08.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f0b21a0f-a4a5-4c35-90cc-dbcc8206da45","head Pod":"stop-zllpq-head-ctt42","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-zllpq-head-ctt42. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:08.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f0b21a0f-a4a5-4c35-90cc-dbcc8206da45","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:47:08.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f0b21a0f-a4a5-4c35-90cc-dbcc8206da45","worker Pod":"stop-zllpq-small-group-worker-bfxtm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-zllpq-small-group-worker-bfxtm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:08.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f0b21a0f-a4a5-4c35-90cc-dbcc8206da45","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:08.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f0b21a0f-a4a5-4c35-90cc-dbcc8206da45","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:47:08.437Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f0b21a0f-a4a5-4c35-90cc-dbcc8206da45","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:47:08.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f0b21a0f-a4a5-4c35-90cc-dbcc8206da45","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:08.437Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f0b21a0f-a4a5-4c35-90cc-dbcc8206da45","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:47:08.437Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"f0b21a0f-a4a5-4c35-90cc-dbcc8206da45","seconds":300}
{"level":"info","ts":"2026-09-18T09:47:08.883Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"b6e40bf2-5ffb-4e00-9f16-128b14f2304d","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:08.883Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"b6e40bf2-5ffb-4e00-9f16-128b14f2304d","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:08.883Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"b6e40bf2-5ffb-4e00-9f16-128b14f2304d","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:08.884Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"b6e40bf2-5ffb-4e00-9f16-128b14f2304d"}
{"level":"info","ts":"2026-09-18T09:47:08.884Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"b6e40bf2-5ffb-4e00-9f16-128b14f2304d","rayCluster":"stop-zllpq","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:47:08.887Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"b6e40bf2-5ffb-4e00-9f16-128b14f2304d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:08.887Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"b6e40bf2-5ffb-4e00-9f16-128b14f2304d","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:47:08.902Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"54e98bf6-209e-49e5-8732-45868464c62f","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:08.902Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"54e98bf6-209e-49e5-8732-45868464c62f","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:08.902Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"54e98bf6-209e-49e5-8732-45868464c62f","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:08.902Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"54e98bf6-209e-49e5-8732-45868464c62f"}
{"level":"info","ts":"2026-09-18T09:47:08.902Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"54e98bf6-209e-49e5-8732-45868464c62f","rayCluster":"stop-zllpq","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:47:08.904Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"54e98bf6-209e-49e5-8732-45868464c62f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:11.902Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"bc775f18-c98d-4da7-9509-cede7930384d","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:11.902Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"bc775f18-c98d-4da7-9509-cede7930384d","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:11.902Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"bc775f18-c98d-4da7-9509-cede7930384d","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:11.903Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"bc775f18-c98d-4da7-9509-cede7930384d"}
{"level":"info","ts":"2026-09-18T09:47:11.903Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"bc775f18-c98d-4da7-9509-cede7930384d","rayCluster":"stop-zllpq","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:47:11.906Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"bc775f18-c98d-4da7-9509-cede7930384d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:14.907Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"c07ccc6e-0c35-4c7d-85fd-821c73b8aaf7","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:14.907Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"c07ccc6e-0c35-4c7d-85fd-821c73b8aaf7","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:14.907Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"c07ccc6e-0c35-4c7d-85fd-821c73b8aaf7","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:14.907Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"c07ccc6e-0c35-4c7d-85fd-821c73b8aaf7"}
{"level":"info","ts":"2026-09-18T09:47:14.907Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"c07ccc6e-0c35-4c7d-85fd-821c73b8aaf7","rayCluster":"stop-zllpq","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:47:14.910Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"c07ccc6e-0c35-4c7d-85fd-821c73b8aaf7","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:17.911Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"18d51e24-a80c-402d-be56-8cc45a258c63","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:17.911Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"18d51e24-a80c-402d-be56-8cc45a258c63","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:17.911Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"18d51e24-a80c-402d-be56-8cc45a258c63","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:17.912Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"18d51e24-a80c-402d-be56-8cc45a258c63"}
{"level":"info","ts":"2026-09-18T09:47:17.912Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"18d51e24-a80c-402d-be56-8cc45a258c63","rayCluster":"stop-zllpq","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:47:17.915Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"18d51e24-a80c-402d-be56-8cc45a258c63","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:20.915Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"1680ae8f-d018-4306-ac60-beccfce5bbc8","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:20.915Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"1680ae8f-d018-4306-ac60-beccfce5bbc8","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:20.915Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"1680ae8f-d018-4306-ac60-beccfce5bbc8","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:20.915Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"1680ae8f-d018-4306-ac60-beccfce5bbc8"}
{"level":"info","ts":"2026-09-18T09:47:20.915Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"1680ae8f-d018-4306-ac60-beccfce5bbc8","rayCluster":"stop-zllpq","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:47:20.919Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"1680ae8f-d018-4306-ac60-beccfce5bbc8","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:23.919Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"e8921316-3f72-484b-9b79-b7233a75d37c","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:23.919Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"e8921316-3f72-484b-9b79-b7233a75d37c","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:23.919Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"e8921316-3f72-484b-9b79-b7233a75d37c","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:23.919Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"e8921316-3f72-484b-9b79-b7233a75d37c"}
{"level":"info","ts":"2026-09-18T09:47:23.919Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"e8921316-3f72-484b-9b79-b7233a75d37c","rayCluster":"stop-zllpq","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:47:23.922Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"e8921316-3f72-484b-9b79-b7233a75d37c","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:26.922Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"758bc318-b160-4a71-b328-4eef89767d01","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:26.922Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"758bc318-b160-4a71-b328-4eef89767d01","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:26.922Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"758bc318-b160-4a71-b328-4eef89767d01","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:26.923Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"758bc318-b160-4a71-b328-4eef89767d01"}
{"level":"info","ts":"2026-09-18T09:47:26.923Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"758bc318-b160-4a71-b328-4eef89767d01","rayCluster":"stop-zllpq","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:47:26.926Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"758bc318-b160-4a71-b328-4eef89767d01","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:29.780Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"4465950c-d175-49d5-9d49-e82663a0cd49","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:47:29.780Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"4465950c-d175-49d5-9d49-e82663a0cd49","pod":"fail-bcffx-head-s6qj5","ip":"10.133.0.29"}
{"level":"info","ts":"2026-09-18T09:47:29.780Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"4465950c-d175-49d5-9d49-e82663a0cd49","pod":"fail-bcffx-small-group-worker-dcw8j","ip":"10.132.0.31"}
{"level":"info","ts":"2026-09-18T09:47:29.780Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"4465950c-d175-49d5-9d49-e82663a0cd49","rayCluster":"fail-bcffx","podIPs":["10.133.0.29","10.132.0.31"]}
{"level":"info","ts":"2026-09-18T09:47:29.927Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"98d47d09-a5ad-4eb5-8413-00de9e57f833","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:29.927Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"98d47d09-a5ad-4eb5-8413-00de9e57f833","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:29.927Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"98d47d09-a5ad-4eb5-8413-00de9e57f833","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:29.927Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"98d47d09-a5ad-4eb5-8413-00de9e57f833"}
{"level":"info","ts":"2026-09-18T09:47:29.927Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"98d47d09-a5ad-4eb5-8413-00de9e57f833","rayCluster":"stop-zllpq","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:47:29.930Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"98d47d09-a5ad-4eb5-8413-00de9e57f833","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:47:06Z","endTime":"2026-09-18T09:47:29Z"},"jobId":"stop-nmzdw","rayClusterName":"stop-zllpq","dashboardURL":"stop-zllpq-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Complete","message":"Job was intentionally stopped.","startTime":"2026-09-18T09:46:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:06Z","stateTransitionTimes":{"ready":"2026-09-18T09:47:06Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"stop-zllpq-head-ctt42","serviceName":"stop-zllpq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:56Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:47:06Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:46:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:46:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:29.930Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"98d47d09-a5ad-4eb5-8413-00de9e57f833","old JobStatus":"RUNNING","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-09-18T09:47:29.944Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"08041a17-0295-4724-812c-b6067b58f717","JobStatus":"STOPPED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:30.173Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"ab406586-ad51-478e-9f98-d0e44d3807f7","DeletionTimestamp":"2026-09-18 09:47:30 +0000 UTC"}
{"level":"info","ts":"2026-09-18T09:47:30.173Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"ab406586-ad51-478e-9f98-d0e44d3807f7","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:47:30.202Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"29c63bac-eccf-4dea-9a77-60582a19666c","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:30.202Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"29c63bac-eccf-4dea-9a77-60582a19666c","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:47:30.226Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"29c63bac-eccf-4dea-9a77-60582a19666c"}
{"level":"info","ts":"2026-09-18T09:47:30.226Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"29c63bac-eccf-4dea-9a77-60582a19666c","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:47:30.226Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"29c63bac-eccf-4dea-9a77-60582a19666c","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:47:30.226Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"29c63bac-eccf-4dea-9a77-60582a19666c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:47:30.235Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"6d6b4c6e-6a4f-4feb-811c-1c9f3d0fb0b8","rayCluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:30.235Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"cf5ee742-10be-4c56-8a23-620b77f03ab3","namespacedName":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:30.235Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"cf5ee742-10be-4c56-8a23-620b77f03ab3","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:30.235Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"cf5ee742-10be-4c56-8a23-620b77f03ab3","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:30.235Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"6d6b4c6e-6a4f-4feb-811c-1c9f3d0fb0b8","pod":"stop-zllpq-small-group-worker-bfxtm","ip":"10.133.0.30"}
{"level":"info","ts":"2026-09-18T09:47:30.235Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"6d6b4c6e-6a4f-4feb-811c-1c9f3d0fb0b8","pod":"stop-zllpq-head-ctt42","ip":"10.132.0.32"}
{"level":"info","ts":"2026-09-18T09:47:30.235Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"cf5ee742-10be-4c56-8a23-620b77f03ab3","namespace":"test-ns-q2lj2","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-09-18T09:47:30.235Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"cf5ee742-10be-4c56-8a23-620b77f03ab3","name":"kuberay-gateway-access","namespace":"test-ns-q2lj2","remainingClusters":1}
{"level":"info","ts":"2026-09-18T09:47:30.235Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"6d6b4c6e-6a4f-4feb-811c-1c9f3d0fb0b8","rayCluster":"stop-zllpq","podIPs":["10.133.0.30","10.132.0.32"]}
{"level":"info","ts":"2026-09-18T09:47:30.235Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"reconcileID":"1bcdaec0-af6e-4db7-aff9-04a33bee0256"}
{"level":"info","ts":"2026-09-18T09:47:30.235Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"4e71f2a0-549b-44bc-9240-b129d9975fb8"}
{"level":"info","ts":"2026-09-18T09:47:30.246Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"e44715ee-ae47-4590-9b9e-2b1a9dd29379"}
{"level":"info","ts":"2026-09-18T09:47:30.246Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"58ef8a62-37aa-404c-9159-8ed3468b8304","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:30.246Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"58ef8a62-37aa-404c-9159-8ed3468b8304"}
{"level":"info","ts":"2026-09-18T09:47:30.246Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"58ef8a62-37aa-404c-9159-8ed3468b8304","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:47:30.246Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"58ef8a62-37aa-404c-9159-8ed3468b8304","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-fg9xm","rayClusterName":"counter-auth-bxs8j","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:47:30.246Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"58ef8a62-37aa-404c-9159-8ed3468b8304","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:47:30.255Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"cf5ee742-10be-4c56-8a23-620b77f03ab3","name":"test-ns-q2lj2-stop-zllpq","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T09:47:30.256Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"58ef8a62-37aa-404c-9159-8ed3468b8304","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter-auth\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T09:47:30.256Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"58ef8a62-37aa-404c-9159-8ed3468b8304"}
{"level":"error","ts":"2026-09-18T09:47:30.256Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"58ef8a62-37aa-404c-9159-8ed3468b8304","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.256Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"9aa1150e-bb4f-46c9-a99f-15e98c5917a2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:30.256Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"9aa1150e-bb4f-46c9-a99f-15e98c5917a2","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.256Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"9aa1150e-bb4f-46c9-a99f-15e98c5917a2","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:30.256Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"9aa1150e-bb4f-46c9-a99f-15e98c5917a2","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:30.269Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"cf5ee742-10be-4c56-8a23-620b77f03ab3","configMap":"kube-rbac-proxy-config-stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:30.272Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.272Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:47:30.278Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"cf5ee742-10be-4c56-8a23-620b77f03ab3","serviceAccount":"stop-zllpq-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:47:30.286Z","logger":"raycluster-resource","msg":"default","name":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:30.286Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-zllpq","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"9aa1150e-bb4f-46c9-a99f-15e98c5917a2","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"642aa103-031e-4e71-a698-e9157939394a"}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"642aa103-031e-4e71-a698-e9157939394a"}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"642aa103-031e-4e71-a698-e9157939394a"}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"9aa1150e-bb4f-46c9-a99f-15e98c5917a2"}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"9aa1150e-bb4f-46c9-a99f-15e98c5917a2","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"9aa1150e-bb4f-46c9-a99f-15e98c5917a2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"99ce3251-ecdb-4213-87f6-1b4c39645ca5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"99ce3251-ecdb-4213-87f6-1b4c39645ca5","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"99ce3251-ecdb-4213-87f6-1b4c39645ca5","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"c77715bd-790d-4bfd-92f5-65643459c215","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"c77715bd-790d-4bfd-92f5-65643459c215","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"99ce3251-ecdb-4213-87f6-1b4c39645ca5"}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"99ce3251-ecdb-4213-87f6-1b4c39645ca5","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"99ce3251-ecdb-4213-87f6-1b4c39645ca5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"f90b3618-de0c-4f08-a4b5-c6b36149c172","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"f90b3618-de0c-4f08-a4b5-c6b36149c172","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"f90b3618-de0c-4f08-a4b5-c6b36149c172","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"8f5365ba-9e5f-4d0a-ae1e-049a3db01b40","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"f90b3618-de0c-4f08-a4b5-c6b36149c172"}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"f90b3618-de0c-4f08-a4b5-c6b36149c172","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:30.295Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"f90b3618-de0c-4f08-a4b5-c6b36149c172","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:47:30.299Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"cf5ee742-10be-4c56-8a23-620b77f03ab3","cluster":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:30.299Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7e1fe222-df13-4fd5-a8d6-5665357e2a80","namespacedName":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:30.299Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7e1fe222-df13-4fd5-a8d6-5665357e2a80","mode":"IntegratedOAuth","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.299Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7e1fe222-df13-4fd5-a8d6-5665357e2a80","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.299Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7e1fe222-df13-4fd5-a8d6-5665357e2a80","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.299Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"ff6f0a8a-de47-4694-b061-46aae7a60b29"}
{"level":"info","ts":"2026-09-18T09:47:30.305Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.305Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:47:30.307Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"c77715bd-790d-4bfd-92f5-65643459c215","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.307Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"c77715bd-790d-4bfd-92f5-65643459c215","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.309Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"642aa103-031e-4e71-a698-e9157939394a","name":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:30.309Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"642aa103-031e-4e71-a698-e9157939394a","error":"head certificate secret ray-head-secret-counter-auth-6hjnc not found"}
{"level":"error","ts":"2026-09-18T09:47:30.309Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"642aa103-031e-4e71-a698-e9157939394a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.309Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"642aa103-031e-4e71-a698-e9157939394a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found"}
{"level":"info","ts":"2026-09-18T09:47:30.309Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"642aa103-031e-4e71-a698-e9157939394a","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.309Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"642aa103-031e-4e71-a698-e9157939394a","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.309Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"642aa103-031e-4e71-a698-e9157939394a","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.309Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"642aa103-031e-4e71-a698-e9157939394a","name":"counter-auth-6hjnc","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:47:30.315Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"8f5365ba-9e5f-4d0a-ae1e-049a3db01b40","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.315Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"8813d058-45f2-4d17-9711-3aee8868b93a"}
{"level":"info","ts":"2026-09-18T09:47:30.315Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"23185d01-6be3-44c5-b293-3baf470715a7","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.320Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"999256e4-a7cf-49d5-ae0a-3c517ad1a892","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:30.320Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"999256e4-a7cf-49d5-ae0a-3c517ad1a892","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.320Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"999256e4-a7cf-49d5-ae0a-3c517ad1a892","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:30.320Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"999256e4-a7cf-49d5-ae0a-3c517ad1a892"}
{"level":"info","ts":"2026-09-18T09:47:30.320Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"999256e4-a7cf-49d5-ae0a-3c517ad1a892","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:30.320Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"999256e4-a7cf-49d5-ae0a-3c517ad1a892","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:47:30.366Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"c77715bd-790d-4bfd-92f5-65643459c215","rayCluster":"counter-auth-6hjnc","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T09:47:30.366Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"c77715bd-790d-4bfd-92f5-65643459c215","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.367Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"642aa103-031e-4e71-a698-e9157939394a","name":"counter-auth-6hjnc","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-6hjnc\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc"}}
{"level":"info","ts":"2026-09-18T09:47:30.367Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"642aa103-031e-4e71-a698-e9157939394a"}
{"level":"error","ts":"2026-09-18T09:47:30.367Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"642aa103-031e-4e71-a698-e9157939394a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.367Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"554d2773-7019-4ba5-8203-89d81584c477"}
{"level":"info","ts":"2026-09-18T09:47:30.367Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"554d2773-7019-4ba5-8203-89d81584c477"}
{"level":"info","ts":"2026-09-18T09:47:30.367Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"554d2773-7019-4ba5-8203-89d81584c477"}
{"level":"info","ts":"2026-09-18T09:47:30.367Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"554d2773-7019-4ba5-8203-89d81584c477","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:30.367Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"554d2773-7019-4ba5-8203-89d81584c477","error":"head certificate secret ray-head-secret-counter-auth-6hjnc not found"}
{"level":"error","ts":"2026-09-18T09:47:30.367Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"554d2773-7019-4ba5-8203-89d81584c477","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.367Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"554d2773-7019-4ba5-8203-89d81584c477","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found"}
{"level":"info","ts":"2026-09-18T09:47:30.367Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"554d2773-7019-4ba5-8203-89d81584c477","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.367Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"554d2773-7019-4ba5-8203-89d81584c477","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.367Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"554d2773-7019-4ba5-8203-89d81584c477","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.367Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"554d2773-7019-4ba5-8203-89d81584c477","name":"counter-auth-6hjnc","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:47:30.372Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"23185d01-6be3-44c5-b293-3baf470715a7","name":"counter-auth-6hjnc-head"}
{"level":"info","ts":"2026-09-18T09:47:30.372Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7e1fe222-df13-4fd5-a8d6-5665357e2a80","name":"counter-auth-6hjnc-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:47:30.376Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"c77715bd-790d-4bfd-92f5-65643459c215","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.382Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"554d2773-7019-4ba5-8203-89d81584c477"}
{"level":"error","ts":"2026-09-18T09:47:30.382Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"554d2773-7019-4ba5-8203-89d81584c477","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.382Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"ad753fe1-27df-49dd-a20a-864ac0c2df27"}
{"level":"info","ts":"2026-09-18T09:47:30.382Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"ad753fe1-27df-49dd-a20a-864ac0c2df27"}
{"level":"info","ts":"2026-09-18T09:47:30.382Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"ad753fe1-27df-49dd-a20a-864ac0c2df27"}
{"level":"info","ts":"2026-09-18T09:47:30.382Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"ad753fe1-27df-49dd-a20a-864ac0c2df27","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:30.382Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"ad753fe1-27df-49dd-a20a-864ac0c2df27","error":"head certificate secret ray-head-secret-counter-auth-6hjnc not found"}
{"level":"error","ts":"2026-09-18T09:47:30.382Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"ad753fe1-27df-49dd-a20a-864ac0c2df27","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.382Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"ad753fe1-27df-49dd-a20a-864ac0c2df27","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found"}
{"level":"info","ts":"2026-09-18T09:47:30.382Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"ad753fe1-27df-49dd-a20a-864ac0c2df27","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.382Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"ad753fe1-27df-49dd-a20a-864ac0c2df27","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.382Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"ad753fe1-27df-49dd-a20a-864ac0c2df27","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.382Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"ad753fe1-27df-49dd-a20a-864ac0c2df27","name":"counter-auth-6hjnc","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:47:30.382Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7e1fe222-df13-4fd5-a8d6-5665357e2a80","name":"test-ns-q2lj2-counter-auth-6hjnc","operation":"created"}
{"level":"info","ts":"2026-09-18T09:47:30.466Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"a1d268e3-5a41-44c1-9bfd-70444a997830","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:30.466Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"a1d268e3-5a41-44c1-9bfd-70444a997830","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.466Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"a1d268e3-5a41-44c1-9bfd-70444a997830","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:30.466Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"a1d268e3-5a41-44c1-9bfd-70444a997830"}
{"level":"info","ts":"2026-09-18T09:47:30.466Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"23185d01-6be3-44c5-b293-3baf470715a7","name":"counter-auth-6hjnc-workers"}
{"level":"info","ts":"2026-09-18T09:47:30.466Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"23185d01-6be3-44c5-b293-3baf470715a7","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.466Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"60de1579-8288-4cb2-a8ab-43738fe6e931"}
{"level":"info","ts":"2026-09-18T09:47:30.466Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"926fa7e8-2b39-4681-95fe-25614b6bfd3f","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.467Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7e1fe222-df13-4fd5-a8d6-5665357e2a80","name":"kube-rbac-proxy-config-counter-auth-6hjnc","operation":"created"}
{"level":"info","ts":"2026-09-18T09:47:30.467Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"ad753fe1-27df-49dd-a20a-864ac0c2df27","name":"counter-auth-6hjnc","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-6hjnc\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc"}}
{"level":"info","ts":"2026-09-18T09:47:30.467Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"ad753fe1-27df-49dd-a20a-864ac0c2df27"}
{"level":"error","ts":"2026-09-18T09:47:30.467Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"ad753fe1-27df-49dd-a20a-864ac0c2df27","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.467Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"bc167ab5-dd93-4c06-958c-84cf38ac1ce9"}
{"level":"info","ts":"2026-09-18T09:47:30.467Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"bc167ab5-dd93-4c06-958c-84cf38ac1ce9"}
{"level":"info","ts":"2026-09-18T09:47:30.467Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"bc167ab5-dd93-4c06-958c-84cf38ac1ce9"}
{"level":"info","ts":"2026-09-18T09:47:30.467Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"bc167ab5-dd93-4c06-958c-84cf38ac1ce9","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:30.467Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"bc167ab5-dd93-4c06-958c-84cf38ac1ce9","error":"head certificate secret ray-head-secret-counter-auth-6hjnc not found"}
{"level":"error","ts":"2026-09-18T09:47:30.467Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"bc167ab5-dd93-4c06-958c-84cf38ac1ce9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.467Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"bc167ab5-dd93-4c06-958c-84cf38ac1ce9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found"}
{"level":"info","ts":"2026-09-18T09:47:30.467Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"bc167ab5-dd93-4c06-958c-84cf38ac1ce9","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.467Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"bc167ab5-dd93-4c06-958c-84cf38ac1ce9","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.467Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"bc167ab5-dd93-4c06-958c-84cf38ac1ce9","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.467Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"c77715bd-790d-4bfd-92f5-65643459c215","rayCluster":"counter-auth-6hjnc","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:47:30.467Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"bc167ab5-dd93-4c06-958c-84cf38ac1ce9"}
{"level":"error","ts":"2026-09-18T09:47:30.467Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"bc167ab5-dd93-4c06-958c-84cf38ac1ce9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.466Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"a1d268e3-5a41-44c1-9bfd-70444a997830","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:30.468Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"a1d268e3-5a41-44c1-9bfd-70444a997830","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:30.468Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"a1d268e3-5a41-44c1-9bfd-70444a997830","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e18d6b6e-a32b-48b5-b315-c91b992661e4"}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e18d6b6e-a32b-48b5-b315-c91b992661e4"}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e18d6b6e-a32b-48b5-b315-c91b992661e4"}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e18d6b6e-a32b-48b5-b315-c91b992661e4","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e18d6b6e-a32b-48b5-b315-c91b992661e4","error":"head certificate secret ray-head-secret-counter-auth-6hjnc not found"}
{"level":"error","ts":"2026-09-18T09:47:30.566Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e18d6b6e-a32b-48b5-b315-c91b992661e4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e18d6b6e-a32b-48b5-b315-c91b992661e4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found"}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e18d6b6e-a32b-48b5-b315-c91b992661e4","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e18d6b6e-a32b-48b5-b315-c91b992661e4","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e18d6b6e-a32b-48b5-b315-c91b992661e4","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e18d6b6e-a32b-48b5-b315-c91b992661e4"}
{"level":"error","ts":"2026-09-18T09:47:30.566Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e18d6b6e-a32b-48b5-b315-c91b992661e4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:47:30.566Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7e1fe222-df13-4fd5-a8d6-5665357e2a80","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-6hjnc\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7e1fe222-df13-4fd5-a8d6-5665357e2a80"}
{"level":"error","ts":"2026-09-18T09:47:30.566Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7e1fe222-df13-4fd5-a8d6-5665357e2a80","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-6hjnc\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"d1373c3a-0bb0-4132-ab04-c985eeee02ac","namespacedName":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"d1373c3a-0bb0-4132-ab04-c985eeee02ac"}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"d1373c3a-0bb0-4132-ab04-c985eeee02ac","namespace":"test-ns-q2lj2","authClusters":2}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"d1373c3a-0bb0-4132-ab04-c985eeee02ac","namespace":"test-ns-q2lj2","authClusters":2}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"37c79a0d-e00d-4a39-9b3f-a485aaaa2c57","namespacedName":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"37c79a0d-e00d-4a39-9b3f-a485aaaa2c57","mode":"IntegratedOAuth","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.566Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"37c79a0d-e00d-4a39-9b3f-a485aaaa2c57","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.567Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"7968c5ff-8116-4436-9598-2728211cf73a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:30.567Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"7968c5ff-8116-4436-9598-2728211cf73a","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.567Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"7968c5ff-8116-4436-9598-2728211cf73a","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:30.567Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"7968c5ff-8116-4436-9598-2728211cf73a"}
{"level":"info","ts":"2026-09-18T09:47:30.567Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"7968c5ff-8116-4436-9598-2728211cf73a","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:30.567Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"7968c5ff-8116-4436-9598-2728211cf73a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:30.567Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"c77715bd-790d-4bfd-92f5-65643459c215","rayCluster":"counter-auth-6hjnc","podIPs":[]}
{"level":"error","ts":"2026-09-18T09:47:30.567Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"c77715bd-790d-4bfd-92f5-65643459c215","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-auth-6hjnc\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.567Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"c77715bd-790d-4bfd-92f5-65643459c215"}
{"level":"error","ts":"2026-09-18T09:47:30.567Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"c77715bd-790d-4bfd-92f5-65643459c215","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-auth-6hjnc\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.567Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"2cb85f7f-96d6-4ff5-bc90-5eb34d589540","namespace":"test-ns-q2lj2","clusterName":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:30.570Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"926fa7e8-2b39-4681-95fe-25614b6bfd3f","name":"counter-auth-6hjnc-head"}
{"level":"info","ts":"2026-09-18T09:47:30.580Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"37c79a0d-e00d-4a39-9b3f-a485aaaa2c57","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.580Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"254ab1ec-6ea6-4906-897c-aef9b99565e3","namespacedName":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:30.580Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"254ab1ec-6ea6-4906-897c-aef9b99565e3","mode":"IntegratedOAuth","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.580Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"254ab1ec-6ea6-4906-897c-aef9b99565e3","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.580Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b6c4bdbc-9c71-409e-aecd-419fcac881b6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:30.580Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b6c4bdbc-9c71-409e-aecd-419fcac881b6","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.580Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b6c4bdbc-9c71-409e-aecd-419fcac881b6","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:30.586Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"926fa7e8-2b39-4681-95fe-25614b6bfd3f","name":"counter-auth-6hjnc-workers"}
{"level":"info","ts":"2026-09-18T09:47:30.586Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"926fa7e8-2b39-4681-95fe-25614b6bfd3f","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.586Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"4e273dc9-ae21-41a8-826f-2738c4687ee8","cluster":"counter-auth-6hjnc"}
{"level":"error","ts":"2026-09-18T09:47:30.591Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"254ab1ec-6ea6-4906-897c-aef9b99565e3","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-6hjnc\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.591Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"254ab1ec-6ea6-4906-897c-aef9b99565e3"}
{"level":"error","ts":"2026-09-18T09:47:30.591Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"254ab1ec-6ea6-4906-897c-aef9b99565e3","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-6hjnc\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.592Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"2cb85f7f-96d6-4ff5-bc90-5eb34d589540","namespace":"test-ns-q2lj2","clusterName":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:47:30.592Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"1d17fad6-e4cc-4782-a17d-a45a4aa8ad33","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.592Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"1d17fad6-e4cc-4782-a17d-a45a4aa8ad33"}
{"level":"info","ts":"2026-09-18T09:47:30.596Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"51a70c0e-3829-4f78-a73d-d9990fe686a9","namespacedName":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:30.597Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"51a70c0e-3829-4f78-a73d-d9990fe686a9","mode":"IntegratedOAuth","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.597Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"51a70c0e-3829-4f78-a73d-d9990fe686a9","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.666Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b6c4bdbc-9c71-409e-aecd-419fcac881b6"}
{"level":"info","ts":"2026-09-18T09:47:30.666Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b6c4bdbc-9c71-409e-aecd-419fcac881b6","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:30.666Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d4f7ca9e-5868-4b62-a161-44978b6154ef"}
{"level":"info","ts":"2026-09-18T09:47:30.666Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d4f7ca9e-5868-4b62-a161-44978b6154ef"}
{"level":"info","ts":"2026-09-18T09:47:30.666Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b6c4bdbc-9c71-409e-aecd-419fcac881b6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:30.666Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b6c4bdbc-9c71-409e-aecd-419fcac881b6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:47:30.666Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"4e273dc9-ae21-41a8-826f-2738c4687ee8","name":"counter-auth-6hjnc-head"}
{"level":"info","ts":"2026-09-18T09:47:30.666Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d4f7ca9e-5868-4b62-a161-44978b6154ef"}
{"level":"info","ts":"2026-09-18T09:47:30.666Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d4f7ca9e-5868-4b62-a161-44978b6154ef","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:30.666Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d4f7ca9e-5868-4b62-a161-44978b6154ef","error":"head certificate secret ray-head-secret-counter-auth-6hjnc not found"}
{"level":"error","ts":"2026-09-18T09:47:30.666Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d4f7ca9e-5868-4b62-a161-44978b6154ef","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.666Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d4f7ca9e-5868-4b62-a161-44978b6154ef","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found"}
{"level":"info","ts":"2026-09-18T09:47:30.666Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d4f7ca9e-5868-4b62-a161-44978b6154ef","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.666Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d4f7ca9e-5868-4b62-a161-44978b6154ef","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.666Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d4f7ca9e-5868-4b62-a161-44978b6154ef","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.667Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"51a70c0e-3829-4f78-a73d-d9990fe686a9","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.667Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d4f7ca9e-5868-4b62-a161-44978b6154ef"}
{"level":"error","ts":"2026-09-18T09:47:30.667Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d4f7ca9e-5868-4b62-a161-44978b6154ef","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.679Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"4e273dc9-ae21-41a8-826f-2738c4687ee8","name":"counter-auth-6hjnc-workers"}
{"level":"info","ts":"2026-09-18T09:47:30.679Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"4e273dc9-ae21-41a8-826f-2738c4687ee8","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.681Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"416dfbda-3b5d-45c0-8a84-ebc2b9361216","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:30.681Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"416dfbda-3b5d-45c0-8a84-ebc2b9361216","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:30.681Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"416dfbda-3b5d-45c0-8a84-ebc2b9361216","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:30.681Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"416dfbda-3b5d-45c0-8a84-ebc2b9361216"}
{"level":"info","ts":"2026-09-18T09:47:30.681Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"416dfbda-3b5d-45c0-8a84-ebc2b9361216","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:30.681Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"416dfbda-3b5d-45c0-8a84-ebc2b9361216","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:30.827Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9f26060e-48d2-43c5-ba1a-5e779c37a0d7"}
{"level":"info","ts":"2026-09-18T09:47:30.827Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9f26060e-48d2-43c5-ba1a-5e779c37a0d7"}
{"level":"info","ts":"2026-09-18T09:47:30.827Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9f26060e-48d2-43c5-ba1a-5e779c37a0d7"}
{"level":"info","ts":"2026-09-18T09:47:30.827Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9f26060e-48d2-43c5-ba1a-5e779c37a0d7","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:30.828Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9f26060e-48d2-43c5-ba1a-5e779c37a0d7","error":"head certificate secret ray-head-secret-counter-auth-6hjnc not found"}
{"level":"error","ts":"2026-09-18T09:47:30.828Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9f26060e-48d2-43c5-ba1a-5e779c37a0d7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:30.828Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9f26060e-48d2-43c5-ba1a-5e779c37a0d7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found"}
{"level":"info","ts":"2026-09-18T09:47:30.828Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9f26060e-48d2-43c5-ba1a-5e779c37a0d7","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.828Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9f26060e-48d2-43c5-ba1a-5e779c37a0d7","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.828Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9f26060e-48d2-43c5-ba1a-5e779c37a0d7","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:30.828Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9f26060e-48d2-43c5-ba1a-5e779c37a0d7"}
{"level":"error","ts":"2026-09-18T09:47:30.828Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9f26060e-48d2-43c5-ba1a-5e779c37a0d7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:31.148Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"52daa168-a229-40f5-8f22-67a73f31f2ea"}
{"level":"info","ts":"2026-09-18T09:47:31.148Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"52daa168-a229-40f5-8f22-67a73f31f2ea"}
{"level":"info","ts":"2026-09-18T09:47:31.148Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"52daa168-a229-40f5-8f22-67a73f31f2ea"}
{"level":"info","ts":"2026-09-18T09:47:31.148Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"52daa168-a229-40f5-8f22-67a73f31f2ea","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:31.149Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"52daa168-a229-40f5-8f22-67a73f31f2ea","error":"head certificate secret ray-head-secret-counter-auth-6hjnc not found"}
{"level":"error","ts":"2026-09-18T09:47:31.149Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"52daa168-a229-40f5-8f22-67a73f31f2ea","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:31.149Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"52daa168-a229-40f5-8f22-67a73f31f2ea","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found"}
{"level":"info","ts":"2026-09-18T09:47:31.149Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"52daa168-a229-40f5-8f22-67a73f31f2ea","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:31.149Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"52daa168-a229-40f5-8f22-67a73f31f2ea","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:31.149Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"52daa168-a229-40f5-8f22-67a73f31f2ea","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"counter-auth-6hjnc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:47:31.149Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"52daa168-a229-40f5-8f22-67a73f31f2ea"}
{"level":"error","ts":"2026-09-18T09:47:31.149Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"52daa168-a229-40f5-8f22-67a73f31f2ea","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-6hjnc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:31.790Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662"}
{"level":"info","ts":"2026-09-18T09:47:31.790Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662"}
{"level":"info","ts":"2026-09-18T09:47:31.790Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662"}
{"level":"info","ts":"2026-09-18T09:47:31.790Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:31.790Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662"}
{"level":"info","ts":"2026-09-18T09:47:31.790Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:47:31.790Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662"}
{"level":"info","ts":"2026-09-18T09:47:31.790Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","mode":"IntegratedOAuth","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:31.790Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","cluster":"counter-auth-6hjnc","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:47:31.790Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","cluster":"counter-auth-6hjnc","authType":"IntegratedOAuth","serviceAccount":"counter-auth-6hjnc-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:47:31.790Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-auth-6hjnc","ray.io/group":"headgroup","ray.io/identifier":"counter-auth-6hjnc-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:47:31.790Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:47:31.790Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:47:31.790Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:47:31.790Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","enabled":true}
{"level":"info","ts":"2026-09-18T09:47:31.810Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","name":"counter-auth-6hjnc-head-wk2nd"}
{"level":"info","ts":"2026-09-18T09:47:31.810Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:47:31.810Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:31.810Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:47:31.810Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:47:31.810Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:31.810Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:47:31.810Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662"}
{"level":"info","ts":"2026-09-18T09:47:31.810Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","nodeType":"worker","rayStartParams":{"address":"counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:47:31.810Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","rayStartCmd":"ray start  --address=counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:47:31.810Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:47:31.810Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","enabled":true}
{"level":"info","ts":"2026-09-18T09:47:31.830Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","name":"counter-auth-6hjnc-small-group-worker-bhfr4"}
{"level":"info","ts":"2026-09-18T09:47:31.830Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:31.830Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:31.830Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"eee55632-5590-4c3f-a668-1c6e31728662","name":"counter-auth-6hjnc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:47:31.848Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed"}
{"level":"info","ts":"2026-09-18T09:47:31.848Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed"}
{"level":"info","ts":"2026-09-18T09:47:31.848Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed"}
{"level":"info","ts":"2026-09-18T09:47:31.848Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:31.848Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:47:31.848Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed","name":"counter-auth-6hjnc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b784c64d-a472-49c9-a0b8-0c8e56dfb2ba","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"38c3ba32-623a-4ba2-b789-6ad3920a5472","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b784c64d-a472-49c9-a0b8-0c8e56dfb2ba","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b784c64d-a472-49c9-a0b8-0c8e56dfb2ba","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"38c3ba32-623a-4ba2-b789-6ad3920a5472"}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"f382e9ae-c268-4cdd-bed2-26feabb0efa2","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b784c64d-a472-49c9-a0b8-0c8e56dfb2ba"}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b784c64d-a472-49c9-a0b8-0c8e56dfb2ba","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:31.849Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b784c64d-a472-49c9-a0b8-0c8e56dfb2ba","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:31.865Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b784c64d-a472-49c9-a0b8-0c8e56dfb2ba","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:47:31.856Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed","name":"counter-auth-6hjnc","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-6hjnc\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc"}}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed"}
{"level":"error","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"f0520ff8-2e7b-4658-9517-bd85f02108ed","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-6hjnc\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c0afdf7-ed3a-4ac9-b9b9-5bcf52b8e27e"}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c0afdf7-ed3a-4ac9-b9b9-5bcf52b8e27e"}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c0afdf7-ed3a-4ac9-b9b9-5bcf52b8e27e"}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c0afdf7-ed3a-4ac9-b9b9-5bcf52b8e27e","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c0afdf7-ed3a-4ac9-b9b9-5bcf52b8e27e","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c0afdf7-ed3a-4ac9-b9b9-5bcf52b8e27e","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c0afdf7-ed3a-4ac9-b9b9-5bcf52b8e27e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c0afdf7-ed3a-4ac9-b9b9-5bcf52b8e27e","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c0afdf7-ed3a-4ac9-b9b9-5bcf52b8e27e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c0afdf7-ed3a-4ac9-b9b9-5bcf52b8e27e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c0afdf7-ed3a-4ac9-b9b9-5bcf52b8e27e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c0afdf7-ed3a-4ac9-b9b9-5bcf52b8e27e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c0afdf7-ed3a-4ac9-b9b9-5bcf52b8e27e","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c0afdf7-ed3a-4ac9-b9b9-5bcf52b8e27e","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c0afdf7-ed3a-4ac9-b9b9-5bcf52b8e27e","name":"counter-auth-6hjnc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:47:31.866Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"f382e9ae-c268-4cdd-bed2-26feabb0efa2","name":"counter-auth-6hjnc-head"}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"135ef398-de94-414e-ba25-470aaa7f10d3","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"07e266d4-a1b2-44b8-a13a-14a2feef8125"}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"07e266d4-a1b2-44b8-a13a-14a2feef8125"}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"07e266d4-a1b2-44b8-a13a-14a2feef8125"}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"135ef398-de94-414e-ba25-470aaa7f10d3"}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"07e266d4-a1b2-44b8-a13a-14a2feef8125","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"07e266d4-a1b2-44b8-a13a-14a2feef8125","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"07e266d4-a1b2-44b8-a13a-14a2feef8125","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"07e266d4-a1b2-44b8-a13a-14a2feef8125","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"07e266d4-a1b2-44b8-a13a-14a2feef8125","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"07e266d4-a1b2-44b8-a13a-14a2feef8125","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"07e266d4-a1b2-44b8-a13a-14a2feef8125","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"07e266d4-a1b2-44b8-a13a-14a2feef8125","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"07e266d4-a1b2-44b8-a13a-14a2feef8125","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"07e266d4-a1b2-44b8-a13a-14a2feef8125","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"07e266d4-a1b2-44b8-a13a-14a2feef8125","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"07e266d4-a1b2-44b8-a13a-14a2feef8125","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:47:31.883Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"07e266d4-a1b2-44b8-a13a-14a2feef8125","seconds":300}
{"level":"info","ts":"2026-09-18T09:47:31.887Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"f382e9ae-c268-4cdd-bed2-26feabb0efa2","name":"counter-auth-6hjnc-workers"}
{"level":"info","ts":"2026-09-18T09:47:31.887Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"f382e9ae-c268-4cdd-bed2-26feabb0efa2","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:31.887Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"3464bff1-0372-42d9-a7b4-2655fc003065","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:31.967Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"04f90645-62d1-4b2b-8d22-3b40614e5f93","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:31.967Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"04f90645-62d1-4b2b-8d22-3b40614e5f93","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:31.967Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"04f90645-62d1-4b2b-8d22-3b40614e5f93","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:31.967Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"04f90645-62d1-4b2b-8d22-3b40614e5f93"}
{"level":"info","ts":"2026-09-18T09:47:31.967Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"04f90645-62d1-4b2b-8d22-3b40614e5f93","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:31.967Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"04f90645-62d1-4b2b-8d22-3b40614e5f93","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:31.967Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"04f90645-62d1-4b2b-8d22-3b40614e5f93","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:47:31.968Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"1034c9f4-2d0e-4164-9157-c99af1999d7a"}
{"level":"info","ts":"2026-09-18T09:47:31.968Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"1034c9f4-2d0e-4164-9157-c99af1999d7a"}
{"level":"info","ts":"2026-09-18T09:47:31.968Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"1034c9f4-2d0e-4164-9157-c99af1999d7a"}
{"level":"info","ts":"2026-09-18T09:47:31.968Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"1034c9f4-2d0e-4164-9157-c99af1999d7a","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:31.968Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"1034c9f4-2d0e-4164-9157-c99af1999d7a","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:47:31.968Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"1034c9f4-2d0e-4164-9157-c99af1999d7a","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:31.968Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"1034c9f4-2d0e-4164-9157-c99af1999d7a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:47:31.968Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"1034c9f4-2d0e-4164-9157-c99af1999d7a","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:31.968Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"1034c9f4-2d0e-4164-9157-c99af1999d7a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:31.968Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"1034c9f4-2d0e-4164-9157-c99af1999d7a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:47:31.968Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"1034c9f4-2d0e-4164-9157-c99af1999d7a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:47:31.968Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"1034c9f4-2d0e-4164-9157-c99af1999d7a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:31.969Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"1034c9f4-2d0e-4164-9157-c99af1999d7a","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:31.969Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"1034c9f4-2d0e-4164-9157-c99af1999d7a","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:31.969Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"1034c9f4-2d0e-4164-9157-c99af1999d7a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:47:31.969Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"1034c9f4-2d0e-4164-9157-c99af1999d7a","seconds":300}
{"level":"info","ts":"2026-09-18T09:47:32.067Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"62f54c24-8bab-4b54-87fd-2210d228a5c6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:32.067Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"62f54c24-8bab-4b54-87fd-2210d228a5c6","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:32.067Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"62f54c24-8bab-4b54-87fd-2210d228a5c6","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:32.068Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"62f54c24-8bab-4b54-87fd-2210d228a5c6"}
{"level":"info","ts":"2026-09-18T09:47:32.068Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"62f54c24-8bab-4b54-87fd-2210d228a5c6","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:32.068Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"62f54c24-8bab-4b54-87fd-2210d228a5c6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:32.072Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"3464bff1-0372-42d9-a7b4-2655fc003065","name":"counter-auth-6hjnc-head"}
{"level":"info","ts":"2026-09-18T09:47:32.087Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"3464bff1-0372-42d9-a7b4-2655fc003065","name":"counter-auth-6hjnc-workers"}
{"level":"info","ts":"2026-09-18T09:47:32.087Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"3464bff1-0372-42d9-a7b4-2655fc003065","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:32.414Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c067bc4-9567-4993-9f3f-ee84efe7ddc7"}
{"level":"info","ts":"2026-09-18T09:47:32.414Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c067bc4-9567-4993-9f3f-ee84efe7ddc7"}
{"level":"info","ts":"2026-09-18T09:47:32.414Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c067bc4-9567-4993-9f3f-ee84efe7ddc7"}
{"level":"info","ts":"2026-09-18T09:47:32.414Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c067bc4-9567-4993-9f3f-ee84efe7ddc7","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:32.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c067bc4-9567-4993-9f3f-ee84efe7ddc7","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:47:32.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c067bc4-9567-4993-9f3f-ee84efe7ddc7","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:32.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c067bc4-9567-4993-9f3f-ee84efe7ddc7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:47:32.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c067bc4-9567-4993-9f3f-ee84efe7ddc7","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:32.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c067bc4-9567-4993-9f3f-ee84efe7ddc7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:32.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c067bc4-9567-4993-9f3f-ee84efe7ddc7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:47:32.414Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c067bc4-9567-4993-9f3f-ee84efe7ddc7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:47:32.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c067bc4-9567-4993-9f3f-ee84efe7ddc7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:32.414Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c067bc4-9567-4993-9f3f-ee84efe7ddc7","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:32.414Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c067bc4-9567-4993-9f3f-ee84efe7ddc7","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:32.415Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c067bc4-9567-4993-9f3f-ee84efe7ddc7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:47:32.415Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"7c067bc4-9567-4993-9f3f-ee84efe7ddc7","seconds":300}
{"level":"info","ts":"2026-09-18T09:47:32.438Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"695dbaf1-584d-4460-854c-039c8f0f7054"}
{"level":"info","ts":"2026-09-18T09:47:32.438Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"695dbaf1-584d-4460-854c-039c8f0f7054"}
{"level":"info","ts":"2026-09-18T09:47:32.438Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"695dbaf1-584d-4460-854c-039c8f0f7054"}
{"level":"info","ts":"2026-09-18T09:47:32.438Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"695dbaf1-584d-4460-854c-039c8f0f7054","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:32.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"695dbaf1-584d-4460-854c-039c8f0f7054","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:47:32.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"695dbaf1-584d-4460-854c-039c8f0f7054","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:32.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"695dbaf1-584d-4460-854c-039c8f0f7054","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:47:32.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"695dbaf1-584d-4460-854c-039c8f0f7054","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:32.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"695dbaf1-584d-4460-854c-039c8f0f7054","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:32.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"695dbaf1-584d-4460-854c-039c8f0f7054","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:47:32.438Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"695dbaf1-584d-4460-854c-039c8f0f7054","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:47:32.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"695dbaf1-584d-4460-854c-039c8f0f7054","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:32.439Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"695dbaf1-584d-4460-854c-039c8f0f7054","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:32.439Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"695dbaf1-584d-4460-854c-039c8f0f7054","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:32.439Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"695dbaf1-584d-4460-854c-039c8f0f7054","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:47:32.439Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"695dbaf1-584d-4460-854c-039c8f0f7054","seconds":300}
{"level":"info","ts":"2026-09-18T09:47:32.574Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"3352e06c-0beb-4a89-a05f-b898fbbf4c84"}
{"level":"info","ts":"2026-09-18T09:47:32.574Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"3352e06c-0beb-4a89-a05f-b898fbbf4c84"}
{"level":"info","ts":"2026-09-18T09:47:32.574Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"3352e06c-0beb-4a89-a05f-b898fbbf4c84"}
{"level":"info","ts":"2026-09-18T09:47:32.574Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"3352e06c-0beb-4a89-a05f-b898fbbf4c84","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:32.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"3352e06c-0beb-4a89-a05f-b898fbbf4c84","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:47:32.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"3352e06c-0beb-4a89-a05f-b898fbbf4c84","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:32.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"3352e06c-0beb-4a89-a05f-b898fbbf4c84","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:47:32.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"3352e06c-0beb-4a89-a05f-b898fbbf4c84","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:32.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"3352e06c-0beb-4a89-a05f-b898fbbf4c84","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:32.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"3352e06c-0beb-4a89-a05f-b898fbbf4c84","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:47:32.575Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"3352e06c-0beb-4a89-a05f-b898fbbf4c84","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:47:32.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"3352e06c-0beb-4a89-a05f-b898fbbf4c84","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:32.575Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"3352e06c-0beb-4a89-a05f-b898fbbf4c84","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:32.575Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"3352e06c-0beb-4a89-a05f-b898fbbf4c84","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:32.575Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"3352e06c-0beb-4a89-a05f-b898fbbf4c84","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:47:32.575Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"3352e06c-0beb-4a89-a05f-b898fbbf4c84","seconds":300}
{"level":"info","ts":"2026-09-18T09:47:32.755Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"0f86cf4c-85f1-44ea-ba9e-cf1afdb82473"}
{"level":"info","ts":"2026-09-18T09:47:32.755Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"0f86cf4c-85f1-44ea-ba9e-cf1afdb82473"}
{"level":"info","ts":"2026-09-18T09:47:32.755Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"0f86cf4c-85f1-44ea-ba9e-cf1afdb82473"}
{"level":"info","ts":"2026-09-18T09:47:32.755Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"0f86cf4c-85f1-44ea-ba9e-cf1afdb82473","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:32.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"0f86cf4c-85f1-44ea-ba9e-cf1afdb82473","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:47:32.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"0f86cf4c-85f1-44ea-ba9e-cf1afdb82473","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:32.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"0f86cf4c-85f1-44ea-ba9e-cf1afdb82473","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:47:32.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"0f86cf4c-85f1-44ea-ba9e-cf1afdb82473","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:32.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"0f86cf4c-85f1-44ea-ba9e-cf1afdb82473","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:32.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"0f86cf4c-85f1-44ea-ba9e-cf1afdb82473","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:47:32.755Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"0f86cf4c-85f1-44ea-ba9e-cf1afdb82473","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:47:32.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"0f86cf4c-85f1-44ea-ba9e-cf1afdb82473","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:32.756Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"0f86cf4c-85f1-44ea-ba9e-cf1afdb82473","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:32.756Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"0f86cf4c-85f1-44ea-ba9e-cf1afdb82473","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:32.756Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"0f86cf4c-85f1-44ea-ba9e-cf1afdb82473","name":"counter-auth-6hjnc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:47:32.770Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"8e293f88-c214-4e32-a887-6e6ecad775b9","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:32.770Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"e37ee5a9-b9db-42bb-9c96-352193b09c8f","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:32.770Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"e37ee5a9-b9db-42bb-9c96-352193b09c8f","pod":"counter-auth-6hjnc-head-wk2nd","ip":"10.132.0.33"}
{"level":"info","ts":"2026-09-18T09:47:32.770Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"e37ee5a9-b9db-42bb-9c96-352193b09c8f","pod":"counter-auth-6hjnc-small-group-worker-bhfr4","ip":"10.133.0.31"}
{"level":"info","ts":"2026-09-18T09:47:32.770Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"29d4e689-9e1a-4281-983b-bb5b120fe5ec","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:32.770Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"29d4e689-9e1a-4281-983b-bb5b120fe5ec","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:32.770Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"29d4e689-9e1a-4281-983b-bb5b120fe5ec","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:32.770Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"29d4e689-9e1a-4281-983b-bb5b120fe5ec"}
{"level":"info","ts":"2026-09-18T09:47:32.770Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"29d4e689-9e1a-4281-983b-bb5b120fe5ec","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:32.770Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"29d4e689-9e1a-4281-983b-bb5b120fe5ec","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:32.770Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"29d4e689-9e1a-4281-983b-bb5b120fe5ec","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:47:32.782Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"e37ee5a9-b9db-42bb-9c96-352193b09c8f","certificate":"ray-head-cert-counter-auth-6hjnc","dnsNames":["counter-auth-6hjnc-head-svc","localhost","counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc","counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local"],"ipAddresses":["10.132.0.33","10.133.0.31","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:47:32.788Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"8e293f88-c214-4e32-a887-6e6ecad775b9","name":"counter-auth-6hjnc-head"}
{"level":"info","ts":"2026-09-18T09:47:32.789Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"e85e2743-e11b-4d56-9ea3-47d0521d8a03","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:32.789Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"e85e2743-e11b-4d56-9ea3-47d0521d8a03","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:32.789Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"e85e2743-e11b-4d56-9ea3-47d0521d8a03","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:32.790Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"e85e2743-e11b-4d56-9ea3-47d0521d8a03"}
{"level":"info","ts":"2026-09-18T09:47:32.790Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"e85e2743-e11b-4d56-9ea3-47d0521d8a03","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:32.790Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"e85e2743-e11b-4d56-9ea3-47d0521d8a03","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:32.796Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"e37ee5a9-b9db-42bb-9c96-352193b09c8f","certificate":"ray-worker-cert-counter-auth-6hjnc","dnsNames":["counter-auth-6hjnc-worker-svc","localhost","counter-auth-6hjnc-small-group","counter-auth-6hjnc-worker-svc.test-ns-q2lj2.svc","counter-auth-6hjnc-worker-svc.test-ns-q2lj2.svc.cluster.local","*.counter-auth-6hjnc-worker-svc.test-ns-q2lj2.svc","*.counter-auth-6hjnc-worker-svc.test-ns-q2lj2.svc.cluster.local","*-worker-*.test-ns-q2lj2.svc","*-worker-*.test-ns-q2lj2.svc.cluster.local"],"ipAddresses":["10.132.0.33","10.133.0.31","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:47:32.796Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"e37ee5a9-b9db-42bb-9c96-352193b09c8f","rayCluster":"counter-auth-6hjnc","podIPs":["10.132.0.33","10.133.0.31"]}
{"level":"info","ts":"2026-09-18T09:47:32.801Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"8e293f88-c214-4e32-a887-6e6ecad775b9","name":"counter-auth-6hjnc-workers"}
{"level":"info","ts":"2026-09-18T09:47:32.801Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"8e293f88-c214-4e32-a887-6e6ecad775b9","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:32.945Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"stop","namespace":"test-ns-q2lj2"},"reconcileID":"e7815904-7b4e-4865-9f1f-62a43c84fcee"}
{"level":"info","ts":"2026-09-18T09:47:33.296Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d857bf22-555f-4ee5-a823-7687a5181c8f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:33.296Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d857bf22-555f-4ee5-a823-7687a5181c8f","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:33.296Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d857bf22-555f-4ee5-a823-7687a5181c8f","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:33.296Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d857bf22-555f-4ee5-a823-7687a5181c8f"}
{"level":"info","ts":"2026-09-18T09:47:33.296Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d857bf22-555f-4ee5-a823-7687a5181c8f","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:33.296Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d857bf22-555f-4ee5-a823-7687a5181c8f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:33.884Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"b224a86c-0683-4a4f-b83e-9efafd0e003e"}
{"level":"info","ts":"2026-09-18T09:47:33.884Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"b224a86c-0683-4a4f-b83e-9efafd0e003e"}
{"level":"info","ts":"2026-09-18T09:47:33.884Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"b224a86c-0683-4a4f-b83e-9efafd0e003e"}
{"level":"info","ts":"2026-09-18T09:47:33.884Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"b224a86c-0683-4a4f-b83e-9efafd0e003e","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:47:33.884Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"b224a86c-0683-4a4f-b83e-9efafd0e003e","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:47:33.884Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"b224a86c-0683-4a4f-b83e-9efafd0e003e","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:33.884Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"b224a86c-0683-4a4f-b83e-9efafd0e003e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:47:33.884Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"b224a86c-0683-4a4f-b83e-9efafd0e003e","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:47:33.884Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"b224a86c-0683-4a4f-b83e-9efafd0e003e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:33.884Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"b224a86c-0683-4a4f-b83e-9efafd0e003e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:47:33.884Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"b224a86c-0683-4a4f-b83e-9efafd0e003e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:47:33.884Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"b224a86c-0683-4a4f-b83e-9efafd0e003e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:47:33.884Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"b224a86c-0683-4a4f-b83e-9efafd0e003e","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:33.884Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"b224a86c-0683-4a4f-b83e-9efafd0e003e","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:47:33.884Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"b224a86c-0683-4a4f-b83e-9efafd0e003e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:47:33.884Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"b224a86c-0683-4a4f-b83e-9efafd0e003e","seconds":300}
{"level":"info","ts":"2026-09-18T09:47:36.297Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"a9938e0b-7824-440e-a396-75cc4187a7be","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:36.297Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"a9938e0b-7824-440e-a396-75cc4187a7be","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:36.297Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"a9938e0b-7824-440e-a396-75cc4187a7be","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:36.297Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"a9938e0b-7824-440e-a396-75cc4187a7be"}
{"level":"info","ts":"2026-09-18T09:47:36.297Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"a9938e0b-7824-440e-a396-75cc4187a7be","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:36.297Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"a9938e0b-7824-440e-a396-75cc4187a7be","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:39.298Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0e7dbfed-4943-486a-9f0e-28c47c075c9f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:39.298Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0e7dbfed-4943-486a-9f0e-28c47c075c9f","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:39.298Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0e7dbfed-4943-486a-9f0e-28c47c075c9f","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:39.298Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0e7dbfed-4943-486a-9f0e-28c47c075c9f"}
{"level":"info","ts":"2026-09-18T09:47:39.298Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0e7dbfed-4943-486a-9f0e-28c47c075c9f","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:39.298Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0e7dbfed-4943-486a-9f0e-28c47c075c9f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:42.299Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"96dc4998-c7ea-4e3a-8eee-9aec2c4f89d3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:42.299Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"96dc4998-c7ea-4e3a-8eee-9aec2c4f89d3","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:42.299Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"96dc4998-c7ea-4e3a-8eee-9aec2c4f89d3","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:42.299Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"96dc4998-c7ea-4e3a-8eee-9aec2c4f89d3"}
{"level":"info","ts":"2026-09-18T09:47:42.299Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"96dc4998-c7ea-4e3a-8eee-9aec2c4f89d3","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:42.299Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"96dc4998-c7ea-4e3a-8eee-9aec2c4f89d3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:45.300Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0cef6363-6afd-4ced-b97d-eccf302fdbd9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:45.300Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0cef6363-6afd-4ced-b97d-eccf302fdbd9","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:45.300Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0cef6363-6afd-4ced-b97d-eccf302fdbd9","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:45.300Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0cef6363-6afd-4ced-b97d-eccf302fdbd9"}
{"level":"info","ts":"2026-09-18T09:47:45.300Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0cef6363-6afd-4ced-b97d-eccf302fdbd9","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:45.300Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0cef6363-6afd-4ced-b97d-eccf302fdbd9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:48.301Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f35423a-b28c-46a3-bfb2-7b438d88dbaa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:48.301Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f35423a-b28c-46a3-bfb2-7b438d88dbaa","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:48.301Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f35423a-b28c-46a3-bfb2-7b438d88dbaa","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:48.301Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f35423a-b28c-46a3-bfb2-7b438d88dbaa"}
{"level":"info","ts":"2026-09-18T09:47:48.301Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f35423a-b28c-46a3-bfb2-7b438d88dbaa","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:48.301Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f35423a-b28c-46a3-bfb2-7b438d88dbaa","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:51.302Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"cbdd4329-5d8f-4c0e-92b8-15680ed06aa2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:51.302Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"cbdd4329-5d8f-4c0e-92b8-15680ed06aa2","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:51.302Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"cbdd4329-5d8f-4c0e-92b8-15680ed06aa2","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:51.302Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"cbdd4329-5d8f-4c0e-92b8-15680ed06aa2"}
{"level":"info","ts":"2026-09-18T09:47:51.302Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"cbdd4329-5d8f-4c0e-92b8-15680ed06aa2","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:51.302Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"cbdd4329-5d8f-4c0e-92b8-15680ed06aa2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:54.303Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"1a791c15-3dd4-46b1-8c55-b0c3b15f9409","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:54.303Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"1a791c15-3dd4-46b1-8c55-b0c3b15f9409","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:54.303Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"1a791c15-3dd4-46b1-8c55-b0c3b15f9409","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:54.303Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"1a791c15-3dd4-46b1-8c55-b0c3b15f9409"}
{"level":"info","ts":"2026-09-18T09:47:54.303Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"1a791c15-3dd4-46b1-8c55-b0c3b15f9409","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:54.303Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"1a791c15-3dd4-46b1-8c55-b0c3b15f9409","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:47:57.304Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"8a3c715a-8d30-413e-9238-179edd123b0c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:47:57.304Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"8a3c715a-8d30-413e-9238-179edd123b0c","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:47:57.304Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"8a3c715a-8d30-413e-9238-179edd123b0c","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:47:57.304Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"8a3c715a-8d30-413e-9238-179edd123b0c"}
{"level":"info","ts":"2026-09-18T09:47:57.304Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"8a3c715a-8d30-413e-9238-179edd123b0c","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:47:57.304Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"8a3c715a-8d30-413e-9238-179edd123b0c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:00.305Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"25d4917d-2826-45ea-8243-5ee2fbcd2c1d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:00.305Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"25d4917d-2826-45ea-8243-5ee2fbcd2c1d","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:00.305Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"25d4917d-2826-45ea-8243-5ee2fbcd2c1d","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:00.305Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"25d4917d-2826-45ea-8243-5ee2fbcd2c1d"}
{"level":"info","ts":"2026-09-18T09:48:00.305Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"25d4917d-2826-45ea-8243-5ee2fbcd2c1d","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:00.305Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"25d4917d-2826-45ea-8243-5ee2fbcd2c1d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:00.593Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"33c574fd-7b20-436f-bd3a-bfd40a37f8e6","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:00.593Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"33c574fd-7b20-436f-bd3a-bfd40a37f8e6","pod":"counter-auth-6hjnc-small-group-worker-bhfr4","ip":"10.133.0.31"}
{"level":"info","ts":"2026-09-18T09:48:00.593Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"33c574fd-7b20-436f-bd3a-bfd40a37f8e6","pod":"counter-auth-6hjnc-head-wk2nd","ip":"10.132.0.33"}
{"level":"info","ts":"2026-09-18T09:48:00.593Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"33c574fd-7b20-436f-bd3a-bfd40a37f8e6","rayCluster":"counter-auth-6hjnc","podIPs":["10.133.0.31","10.132.0.33"]}
{"level":"info","ts":"2026-09-18T09:48:03.306Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d97ae7a9-9741-430a-937f-214f342d98e9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:03.306Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d97ae7a9-9741-430a-937f-214f342d98e9","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:03.306Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d97ae7a9-9741-430a-937f-214f342d98e9","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:03.306Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d97ae7a9-9741-430a-937f-214f342d98e9"}
{"level":"info","ts":"2026-09-18T09:48:03.306Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d97ae7a9-9741-430a-937f-214f342d98e9","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:03.306Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d97ae7a9-9741-430a-937f-214f342d98e9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:06.096Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"d7231e8a-d43b-4695-842f-93c30c50d4ba","namespace":"test-ns-q2lj2","clusterName":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:48:06.096Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-zllpq","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"stop-zllpq","reconcileID":"d7231e8a-d43b-4695-842f-93c30c50d4ba","namespace":"test-ns-q2lj2","clusterName":"stop-zllpq"}
{"level":"info","ts":"2026-09-18T09:48:06.307Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bba77ec0-d279-4047-9649-30f2074ab4b8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:06.307Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bba77ec0-d279-4047-9649-30f2074ab4b8","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:06.307Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bba77ec0-d279-4047-9649-30f2074ab4b8","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:06.307Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bba77ec0-d279-4047-9649-30f2074ab4b8"}
{"level":"info","ts":"2026-09-18T09:48:06.307Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bba77ec0-d279-4047-9649-30f2074ab4b8","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:06.307Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bba77ec0-d279-4047-9649-30f2074ab4b8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:09.308Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"51598b20-b3c1-4e93-92a5-8e87f0e895f7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:09.308Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"51598b20-b3c1-4e93-92a5-8e87f0e895f7","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:09.308Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"51598b20-b3c1-4e93-92a5-8e87f0e895f7","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:09.308Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"51598b20-b3c1-4e93-92a5-8e87f0e895f7"}
{"level":"info","ts":"2026-09-18T09:48:09.308Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"51598b20-b3c1-4e93-92a5-8e87f0e895f7","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:09.308Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"51598b20-b3c1-4e93-92a5-8e87f0e895f7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:12.309Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"f425a67b-6612-4e58-9e32-b3352aea3e6a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:12.309Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"f425a67b-6612-4e58-9e32-b3352aea3e6a","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:12.309Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"f425a67b-6612-4e58-9e32-b3352aea3e6a","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:12.309Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"f425a67b-6612-4e58-9e32-b3352aea3e6a"}
{"level":"info","ts":"2026-09-18T09:48:12.309Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"f425a67b-6612-4e58-9e32-b3352aea3e6a","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:12.309Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"f425a67b-6612-4e58-9e32-b3352aea3e6a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:15.310Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d9d2deb1-c391-40b8-ba36-2b8b64abe020","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:15.310Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d9d2deb1-c391-40b8-ba36-2b8b64abe020","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:15.310Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d9d2deb1-c391-40b8-ba36-2b8b64abe020","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:15.310Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d9d2deb1-c391-40b8-ba36-2b8b64abe020"}
{"level":"info","ts":"2026-09-18T09:48:15.310Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d9d2deb1-c391-40b8-ba36-2b8b64abe020","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:15.310Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d9d2deb1-c391-40b8-ba36-2b8b64abe020","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:18.311Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"06e63458-5058-44d2-9da0-fc59a27c7f61","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:18.311Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"06e63458-5058-44d2-9da0-fc59a27c7f61","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:18.311Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"06e63458-5058-44d2-9da0-fc59a27c7f61","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:18.311Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"06e63458-5058-44d2-9da0-fc59a27c7f61"}
{"level":"info","ts":"2026-09-18T09:48:18.311Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"06e63458-5058-44d2-9da0-fc59a27c7f61","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:18.311Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"06e63458-5058-44d2-9da0-fc59a27c7f61","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:21.312Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"3d995c0d-2f0b-41d4-8193-9ce209040cc0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:21.312Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"3d995c0d-2f0b-41d4-8193-9ce209040cc0","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:21.312Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"3d995c0d-2f0b-41d4-8193-9ce209040cc0","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:21.312Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"3d995c0d-2f0b-41d4-8193-9ce209040cc0"}
{"level":"info","ts":"2026-09-18T09:48:21.312Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"3d995c0d-2f0b-41d4-8193-9ce209040cc0","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:21.312Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"3d995c0d-2f0b-41d4-8193-9ce209040cc0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:24.313Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"326cd34b-c5e1-4d9a-b27c-98599a2b69bf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:24.313Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"326cd34b-c5e1-4d9a-b27c-98599a2b69bf","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:24.313Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"326cd34b-c5e1-4d9a-b27c-98599a2b69bf","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:24.313Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"326cd34b-c5e1-4d9a-b27c-98599a2b69bf"}
{"level":"info","ts":"2026-09-18T09:48:24.313Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"326cd34b-c5e1-4d9a-b27c-98599a2b69bf","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:24.313Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"326cd34b-c5e1-4d9a-b27c-98599a2b69bf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:27.314Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"8b3c47c3-c057-4835-8087-ede75fb449b9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:27.314Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"8b3c47c3-c057-4835-8087-ede75fb449b9","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:27.314Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"8b3c47c3-c057-4835-8087-ede75fb449b9","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:27.315Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"8b3c47c3-c057-4835-8087-ede75fb449b9"}
{"level":"info","ts":"2026-09-18T09:48:27.315Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"8b3c47c3-c057-4835-8087-ede75fb449b9","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:27.315Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"8b3c47c3-c057-4835-8087-ede75fb449b9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:29.781Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"5fff4010-83ec-4ca9-be34-5b9bdea79ba4","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:48:29.781Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"5fff4010-83ec-4ca9-be34-5b9bdea79ba4","pod":"fail-bcffx-head-s6qj5","ip":"10.133.0.29"}
{"level":"info","ts":"2026-09-18T09:48:29.781Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"5fff4010-83ec-4ca9-be34-5b9bdea79ba4","pod":"fail-bcffx-small-group-worker-dcw8j","ip":"10.132.0.31"}
{"level":"info","ts":"2026-09-18T09:48:29.781Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"5fff4010-83ec-4ca9-be34-5b9bdea79ba4","rayCluster":"fail-bcffx","podIPs":["10.133.0.29","10.132.0.31"]}
{"level":"info","ts":"2026-09-18T09:48:30.315Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"79c4556c-58bd-4e1e-a2b3-102f0c5cda47","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:30.315Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"79c4556c-58bd-4e1e-a2b3-102f0c5cda47","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:30.315Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"79c4556c-58bd-4e1e-a2b3-102f0c5cda47","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:30.315Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"79c4556c-58bd-4e1e-a2b3-102f0c5cda47"}
{"level":"info","ts":"2026-09-18T09:48:30.315Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"79c4556c-58bd-4e1e-a2b3-102f0c5cda47","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:30.315Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"79c4556c-58bd-4e1e-a2b3-102f0c5cda47","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:33.316Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f390248-0b2c-4e5d-941b-4d825c5dc7e8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:33.316Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f390248-0b2c-4e5d-941b-4d825c5dc7e8","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:33.316Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f390248-0b2c-4e5d-941b-4d825c5dc7e8","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:33.316Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f390248-0b2c-4e5d-941b-4d825c5dc7e8"}
{"level":"info","ts":"2026-09-18T09:48:33.316Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f390248-0b2c-4e5d-941b-4d825c5dc7e8","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:33.316Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f390248-0b2c-4e5d-941b-4d825c5dc7e8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:36.317Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"09c29cff-b740-40aa-b6c2-b4c556cc5e1f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:36.317Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"09c29cff-b740-40aa-b6c2-b4c556cc5e1f","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:36.317Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"09c29cff-b740-40aa-b6c2-b4c556cc5e1f","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:36.317Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"09c29cff-b740-40aa-b6c2-b4c556cc5e1f"}
{"level":"info","ts":"2026-09-18T09:48:36.317Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"09c29cff-b740-40aa-b6c2-b4c556cc5e1f","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:36.317Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"09c29cff-b740-40aa-b6c2-b4c556cc5e1f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:37.839Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e0e78b03-25fb-46bc-8b36-357709cd5188"}
{"level":"info","ts":"2026-09-18T09:48:37.839Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e0e78b03-25fb-46bc-8b36-357709cd5188"}
{"level":"info","ts":"2026-09-18T09:48:37.839Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e0e78b03-25fb-46bc-8b36-357709cd5188"}
{"level":"info","ts":"2026-09-18T09:48:37.839Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e0e78b03-25fb-46bc-8b36-357709cd5188","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:48:37.840Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e0e78b03-25fb-46bc-8b36-357709cd5188","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:48:37.840Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e0e78b03-25fb-46bc-8b36-357709cd5188","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:48:37.840Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e0e78b03-25fb-46bc-8b36-357709cd5188","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:48:37.840Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e0e78b03-25fb-46bc-8b36-357709cd5188","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:48:37.840Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e0e78b03-25fb-46bc-8b36-357709cd5188","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:48:37.840Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e0e78b03-25fb-46bc-8b36-357709cd5188","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:48:37.840Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e0e78b03-25fb-46bc-8b36-357709cd5188","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:48:37.840Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e0e78b03-25fb-46bc-8b36-357709cd5188","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:48:37.840Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e0e78b03-25fb-46bc-8b36-357709cd5188","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:48:37.840Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e0e78b03-25fb-46bc-8b36-357709cd5188","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:48:37.840Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e0e78b03-25fb-46bc-8b36-357709cd5188","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:48:37.840Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e0e78b03-25fb-46bc-8b36-357709cd5188","seconds":300}
{"level":"info","ts":"2026-09-18T09:48:38.845Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d47d1ffb-6e94-4c9d-8755-1aa3e0ad62ee"}
{"level":"info","ts":"2026-09-18T09:48:38.845Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d47d1ffb-6e94-4c9d-8755-1aa3e0ad62ee"}
{"level":"info","ts":"2026-09-18T09:48:38.845Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d47d1ffb-6e94-4c9d-8755-1aa3e0ad62ee"}
{"level":"info","ts":"2026-09-18T09:48:38.846Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d47d1ffb-6e94-4c9d-8755-1aa3e0ad62ee","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:48:38.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d47d1ffb-6e94-4c9d-8755-1aa3e0ad62ee","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:48:38.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d47d1ffb-6e94-4c9d-8755-1aa3e0ad62ee","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:48:38.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d47d1ffb-6e94-4c9d-8755-1aa3e0ad62ee","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:48:38.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d47d1ffb-6e94-4c9d-8755-1aa3e0ad62ee","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:48:38.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d47d1ffb-6e94-4c9d-8755-1aa3e0ad62ee","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:48:38.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d47d1ffb-6e94-4c9d-8755-1aa3e0ad62ee","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:48:38.846Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d47d1ffb-6e94-4c9d-8755-1aa3e0ad62ee","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:48:38.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d47d1ffb-6e94-4c9d-8755-1aa3e0ad62ee","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:48:38.846Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d47d1ffb-6e94-4c9d-8755-1aa3e0ad62ee","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:48:38.846Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d47d1ffb-6e94-4c9d-8755-1aa3e0ad62ee","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:48:38.846Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d47d1ffb-6e94-4c9d-8755-1aa3e0ad62ee","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:48:38.846Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"d47d1ffb-6e94-4c9d-8755-1aa3e0ad62ee","seconds":300}
{"level":"info","ts":"2026-09-18T09:48:39.318Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5800c229-def2-492c-af9b-5de121cb4edc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:39.318Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5800c229-def2-492c-af9b-5de121cb4edc","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:39.318Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5800c229-def2-492c-af9b-5de121cb4edc","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:39.318Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5800c229-def2-492c-af9b-5de121cb4edc"}
{"level":"info","ts":"2026-09-18T09:48:39.318Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5800c229-def2-492c-af9b-5de121cb4edc","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:39.318Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5800c229-def2-492c-af9b-5de121cb4edc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:42.320Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"660b2f62-9c9e-468e-ac26-b728cb172d35","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:42.320Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"660b2f62-9c9e-468e-ac26-b728cb172d35","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:42.320Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"660b2f62-9c9e-468e-ac26-b728cb172d35","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:42.320Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"660b2f62-9c9e-468e-ac26-b728cb172d35"}
{"level":"info","ts":"2026-09-18T09:48:42.320Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"660b2f62-9c9e-468e-ac26-b728cb172d35","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:42.320Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"660b2f62-9c9e-468e-ac26-b728cb172d35","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:45.321Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"72b18e98-bbd3-4320-bff6-57c702cd78c7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:45.321Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"72b18e98-bbd3-4320-bff6-57c702cd78c7","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:45.321Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"72b18e98-bbd3-4320-bff6-57c702cd78c7","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:45.321Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"72b18e98-bbd3-4320-bff6-57c702cd78c7"}
{"level":"info","ts":"2026-09-18T09:48:45.321Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"72b18e98-bbd3-4320-bff6-57c702cd78c7","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:45.321Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"72b18e98-bbd3-4320-bff6-57c702cd78c7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:48.106Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"14de9f0d-738b-4379-a56c-67007cce64c5"}
{"level":"info","ts":"2026-09-18T09:48:48.106Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"14de9f0d-738b-4379-a56c-67007cce64c5"}
{"level":"info","ts":"2026-09-18T09:48:48.106Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"14de9f0d-738b-4379-a56c-67007cce64c5"}
{"level":"info","ts":"2026-09-18T09:48:48.106Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"14de9f0d-738b-4379-a56c-67007cce64c5","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:48:48.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"14de9f0d-738b-4379-a56c-67007cce64c5","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:48:48.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"14de9f0d-738b-4379-a56c-67007cce64c5","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:48:48.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"14de9f0d-738b-4379-a56c-67007cce64c5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:48:48.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"14de9f0d-738b-4379-a56c-67007cce64c5","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:48:48.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"14de9f0d-738b-4379-a56c-67007cce64c5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:48:48.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"14de9f0d-738b-4379-a56c-67007cce64c5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:48:48.106Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"14de9f0d-738b-4379-a56c-67007cce64c5","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:48:48.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"14de9f0d-738b-4379-a56c-67007cce64c5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:48:48.106Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"14de9f0d-738b-4379-a56c-67007cce64c5","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:48:48.106Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"14de9f0d-738b-4379-a56c-67007cce64c5","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:48:48.106Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"14de9f0d-738b-4379-a56c-67007cce64c5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:48:48.106Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"14de9f0d-738b-4379-a56c-67007cce64c5","seconds":300}
{"level":"info","ts":"2026-09-18T09:48:48.322Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b776b505-f303-4cae-b542-42cbd516a7fd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:48.322Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b776b505-f303-4cae-b542-42cbd516a7fd","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:48.322Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b776b505-f303-4cae-b542-42cbd516a7fd","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:48.323Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b776b505-f303-4cae-b542-42cbd516a7fd"}
{"level":"info","ts":"2026-09-18T09:48:48.323Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b776b505-f303-4cae-b542-42cbd516a7fd","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:48.323Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b776b505-f303-4cae-b542-42cbd516a7fd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:49.118Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"95a06600-24d9-4d9b-b791-239bd8d0d728"}
{"level":"info","ts":"2026-09-18T09:48:49.118Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"95a06600-24d9-4d9b-b791-239bd8d0d728"}
{"level":"info","ts":"2026-09-18T09:48:49.118Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"95a06600-24d9-4d9b-b791-239bd8d0d728"}
{"level":"info","ts":"2026-09-18T09:48:49.118Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"95a06600-24d9-4d9b-b791-239bd8d0d728","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:48:49.118Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"95a06600-24d9-4d9b-b791-239bd8d0d728","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:48:49.118Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"95a06600-24d9-4d9b-b791-239bd8d0d728","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:48:49.118Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"95a06600-24d9-4d9b-b791-239bd8d0d728","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:48:49.118Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"95a06600-24d9-4d9b-b791-239bd8d0d728","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:48:49.118Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"95a06600-24d9-4d9b-b791-239bd8d0d728","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:48:49.118Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"95a06600-24d9-4d9b-b791-239bd8d0d728","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:48:49.118Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"95a06600-24d9-4d9b-b791-239bd8d0d728","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:48:49.118Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"95a06600-24d9-4d9b-b791-239bd8d0d728","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:48:49.118Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"95a06600-24d9-4d9b-b791-239bd8d0d728","podName":"counter-auth-6hjnc-small-group-worker-bhfr4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:48:49.119Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"95a06600-24d9-4d9b-b791-239bd8d0d728","podName":"counter-auth-6hjnc-small-group-worker-bhfr4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:48:49.119Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"95a06600-24d9-4d9b-b791-239bd8d0d728","name":"counter-auth-6hjnc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:49Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:48:49.131Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"52e67f19-8e24-481c-a19a-d8c64bf3119a","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:49.131Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"e343130f-ce25-4ec1-a286-3029c521e637","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:49.131Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"e343130f-ce25-4ec1-a286-3029c521e637","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:49.131Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"e343130f-ce25-4ec1-a286-3029c521e637","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:49.131Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"e343130f-ce25-4ec1-a286-3029c521e637"}
{"level":"info","ts":"2026-09-18T09:48:49.131Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"e343130f-ce25-4ec1-a286-3029c521e637","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:49.131Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"93dccd22-7386-4547-b0ff-d0fc9e2c0f3e","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:49.131Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"e343130f-ce25-4ec1-a286-3029c521e637","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:47:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:49Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:49.131Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"e343130f-ce25-4ec1-a286-3029c521e637","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:48:49.131Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"93dccd22-7386-4547-b0ff-d0fc9e2c0f3e","pod":"counter-auth-6hjnc-head-wk2nd","ip":"10.132.0.33"}
{"level":"info","ts":"2026-09-18T09:48:49.131Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"93dccd22-7386-4547-b0ff-d0fc9e2c0f3e","pod":"counter-auth-6hjnc-small-group-worker-bhfr4","ip":"10.133.0.31"}
{"level":"info","ts":"2026-09-18T09:48:49.131Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"93dccd22-7386-4547-b0ff-d0fc9e2c0f3e","rayCluster":"counter-auth-6hjnc","podIPs":["10.132.0.33","10.133.0.31"]}
{"level":"info","ts":"2026-09-18T09:48:49.144Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"ac3ecb30-874d-4945-8423-94d99e05321d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:49.144Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"ac3ecb30-874d-4945-8423-94d99e05321d","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:49.144Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"ac3ecb30-874d-4945-8423-94d99e05321d","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:49.144Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"ac3ecb30-874d-4945-8423-94d99e05321d"}
{"level":"info","ts":"2026-09-18T09:48:49.144Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"ac3ecb30-874d-4945-8423-94d99e05321d","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:49.144Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"ac3ecb30-874d-4945-8423-94d99e05321d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:49Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:49Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:49.150Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"52e67f19-8e24-481c-a19a-d8c64bf3119a","name":"counter-auth-6hjnc-head"}
{"level":"info","ts":"2026-09-18T09:48:49.161Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"52e67f19-8e24-481c-a19a-d8c64bf3119a","name":"counter-auth-6hjnc-workers"}
{"level":"info","ts":"2026-09-18T09:48:49.161Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"52e67f19-8e24-481c-a19a-d8c64bf3119a","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:51.131Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e2deb658-55c0-4282-9337-8e900c30157d"}
{"level":"info","ts":"2026-09-18T09:48:51.131Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e2deb658-55c0-4282-9337-8e900c30157d"}
{"level":"info","ts":"2026-09-18T09:48:51.131Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e2deb658-55c0-4282-9337-8e900c30157d"}
{"level":"info","ts":"2026-09-18T09:48:51.131Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e2deb658-55c0-4282-9337-8e900c30157d","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:48:51.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e2deb658-55c0-4282-9337-8e900c30157d","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:48:51.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e2deb658-55c0-4282-9337-8e900c30157d","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:48:51.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e2deb658-55c0-4282-9337-8e900c30157d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:48:51.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e2deb658-55c0-4282-9337-8e900c30157d","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:48:51.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e2deb658-55c0-4282-9337-8e900c30157d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:48:51.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e2deb658-55c0-4282-9337-8e900c30157d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:48:51.131Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e2deb658-55c0-4282-9337-8e900c30157d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:48:51.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e2deb658-55c0-4282-9337-8e900c30157d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:48:51.132Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e2deb658-55c0-4282-9337-8e900c30157d","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:47:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:48:51.132Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e2deb658-55c0-4282-9337-8e900c30157d","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:47:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:48:51.132Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e2deb658-55c0-4282-9337-8e900c30157d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:48:51.132Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"e2deb658-55c0-4282-9337-8e900c30157d","seconds":300}
{"level":"info","ts":"2026-09-18T09:48:51.324Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bb241bfe-069f-422a-ad06-83ee771b39fe","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:51.324Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bb241bfe-069f-422a-ad06-83ee771b39fe","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:51.324Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bb241bfe-069f-422a-ad06-83ee771b39fe","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:51.324Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bb241bfe-069f-422a-ad06-83ee771b39fe"}
{"level":"info","ts":"2026-09-18T09:48:51.324Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bb241bfe-069f-422a-ad06-83ee771b39fe","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:51.324Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bb241bfe-069f-422a-ad06-83ee771b39fe","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:49Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:49Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:52.926Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"be869d31-d9b4-48ec-b7ea-31937fa8209a"}
{"level":"info","ts":"2026-09-18T09:48:52.926Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"be869d31-d9b4-48ec-b7ea-31937fa8209a"}
{"level":"info","ts":"2026-09-18T09:48:52.926Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"be869d31-d9b4-48ec-b7ea-31937fa8209a"}
{"level":"info","ts":"2026-09-18T09:48:52.926Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"be869d31-d9b4-48ec-b7ea-31937fa8209a","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:48:52.927Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"be869d31-d9b4-48ec-b7ea-31937fa8209a","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:48:52.927Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"be869d31-d9b4-48ec-b7ea-31937fa8209a","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:48:52.927Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"be869d31-d9b4-48ec-b7ea-31937fa8209a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:48:52.927Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"be869d31-d9b4-48ec-b7ea-31937fa8209a","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:48:52.927Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"be869d31-d9b4-48ec-b7ea-31937fa8209a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:48:52.927Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"be869d31-d9b4-48ec-b7ea-31937fa8209a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:48:52.927Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"be869d31-d9b4-48ec-b7ea-31937fa8209a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:48:52.927Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"be869d31-d9b4-48ec-b7ea-31937fa8209a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:48:52.927Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"be869d31-d9b4-48ec-b7ea-31937fa8209a","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:47:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:48:52.927Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"be869d31-d9b4-48ec-b7ea-31937fa8209a","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:47:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:48:52.927Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"be869d31-d9b4-48ec-b7ea-31937fa8209a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:48:52.927Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"be869d31-d9b4-48ec-b7ea-31937fa8209a","seconds":300}
{"level":"info","ts":"2026-09-18T09:48:53.922Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"836e77e2-6f17-443d-bc9a-466cc8ae8f97"}
{"level":"info","ts":"2026-09-18T09:48:53.922Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"836e77e2-6f17-443d-bc9a-466cc8ae8f97"}
{"level":"info","ts":"2026-09-18T09:48:53.922Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"836e77e2-6f17-443d-bc9a-466cc8ae8f97"}
{"level":"info","ts":"2026-09-18T09:48:53.922Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"836e77e2-6f17-443d-bc9a-466cc8ae8f97","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:48:53.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"836e77e2-6f17-443d-bc9a-466cc8ae8f97","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:48:53.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"836e77e2-6f17-443d-bc9a-466cc8ae8f97","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:48:53.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"836e77e2-6f17-443d-bc9a-466cc8ae8f97","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:48:53.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"836e77e2-6f17-443d-bc9a-466cc8ae8f97","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:48:53.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"836e77e2-6f17-443d-bc9a-466cc8ae8f97","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:48:53.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"836e77e2-6f17-443d-bc9a-466cc8ae8f97","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:48:53.923Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"836e77e2-6f17-443d-bc9a-466cc8ae8f97","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:48:53.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"836e77e2-6f17-443d-bc9a-466cc8ae8f97","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:48:53.923Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"836e77e2-6f17-443d-bc9a-466cc8ae8f97","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:47:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:48:53.923Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"836e77e2-6f17-443d-bc9a-466cc8ae8f97","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:47:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:48:53.923Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"836e77e2-6f17-443d-bc9a-466cc8ae8f97","name":"counter-auth-6hjnc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:49Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:48:53.936Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"9f6a832c-ddd9-4d9e-80d9-b0dbe61a564e","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:53.936Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"482ada12-33b4-43b9-b503-7af5d6796616","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:53.936Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"482ada12-33b4-43b9-b503-7af5d6796616","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:53.936Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"482ada12-33b4-43b9-b503-7af5d6796616","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:53.936Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"482ada12-33b4-43b9-b503-7af5d6796616"}
{"level":"info","ts":"2026-09-18T09:48:53.936Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"482ada12-33b4-43b9-b503-7af5d6796616","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:53.936Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"3d50d53e-64fe-4047-9ffe-56be085fba8a","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:53.936Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"482ada12-33b4-43b9-b503-7af5d6796616","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:49Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:53.936Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"482ada12-33b4-43b9-b503-7af5d6796616","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:48:53.937Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"3d50d53e-64fe-4047-9ffe-56be085fba8a","pod":"counter-auth-6hjnc-head-wk2nd","ip":"10.132.0.33"}
{"level":"info","ts":"2026-09-18T09:48:53.937Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"3d50d53e-64fe-4047-9ffe-56be085fba8a","pod":"counter-auth-6hjnc-small-group-worker-bhfr4","ip":"10.133.0.31"}
{"level":"info","ts":"2026-09-18T09:48:53.937Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"3d50d53e-64fe-4047-9ffe-56be085fba8a","rayCluster":"counter-auth-6hjnc","podIPs":["10.132.0.33","10.133.0.31"]}
{"level":"info","ts":"2026-09-18T09:48:53.950Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5ea928fe-747e-4479-bc71-846bc044e15a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:53.950Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5ea928fe-747e-4479-bc71-846bc044e15a","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:53.950Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5ea928fe-747e-4479-bc71-846bc044e15a","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:53.950Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5ea928fe-747e-4479-bc71-846bc044e15a"}
{"level":"info","ts":"2026-09-18T09:48:53.950Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5ea928fe-747e-4479-bc71-846bc044e15a","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:53.950Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5ea928fe-747e-4479-bc71-846bc044e15a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:53.950Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"9f6a832c-ddd9-4d9e-80d9-b0dbe61a564e","name":"counter-auth-6hjnc-head"}
{"level":"info","ts":"2026-09-18T09:48:53.963Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"9f6a832c-ddd9-4d9e-80d9-b0dbe61a564e","name":"counter-auth-6hjnc-workers"}
{"level":"info","ts":"2026-09-18T09:48:53.963Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"9f6a832c-ddd9-4d9e-80d9-b0dbe61a564e","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:54.325Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f75c601-8bad-4df9-b2b6-dcdb680b8ff8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:54.325Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f75c601-8bad-4df9-b2b6-dcdb680b8ff8","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:54.325Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f75c601-8bad-4df9-b2b6-dcdb680b8ff8","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:54.325Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f75c601-8bad-4df9-b2b6-dcdb680b8ff8"}
{"level":"info","ts":"2026-09-18T09:48:54.325Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f75c601-8bad-4df9-b2b6-dcdb680b8ff8","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:54.325Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"2f75c601-8bad-4df9-b2b6-dcdb680b8ff8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:48:55.936Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"acdde34c-e821-4b04-acf9-f5433daa34af"}
{"level":"info","ts":"2026-09-18T09:48:55.936Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"acdde34c-e821-4b04-acf9-f5433daa34af"}
{"level":"info","ts":"2026-09-18T09:48:55.936Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"acdde34c-e821-4b04-acf9-f5433daa34af"}
{"level":"info","ts":"2026-09-18T09:48:55.937Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"acdde34c-e821-4b04-acf9-f5433daa34af","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:48:55.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"acdde34c-e821-4b04-acf9-f5433daa34af","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:48:55.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"acdde34c-e821-4b04-acf9-f5433daa34af","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:48:55.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"acdde34c-e821-4b04-acf9-f5433daa34af","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:48:55.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"acdde34c-e821-4b04-acf9-f5433daa34af","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:48:55.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"acdde34c-e821-4b04-acf9-f5433daa34af","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:48:55.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"acdde34c-e821-4b04-acf9-f5433daa34af","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:48:55.937Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"acdde34c-e821-4b04-acf9-f5433daa34af","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:48:55.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"acdde34c-e821-4b04-acf9-f5433daa34af","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:48:55.937Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"acdde34c-e821-4b04-acf9-f5433daa34af","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:47:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:48:55.937Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"acdde34c-e821-4b04-acf9-f5433daa34af","podName":"counter-auth-6hjnc-head-wk2nd","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:47:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:48:55.937Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"acdde34c-e821-4b04-acf9-f5433daa34af","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:48:55.937Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"acdde34c-e821-4b04-acf9-f5433daa34af","seconds":300}
{"level":"info","ts":"2026-09-18T09:48:57.326Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"4f5bf25c-61a8-4975-a00c-176e234bbe61","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:48:57.326Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"4f5bf25c-61a8-4975-a00c-176e234bbe61","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:48:57.326Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"4f5bf25c-61a8-4975-a00c-176e234bbe61","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:48:57.327Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"4f5bf25c-61a8-4975-a00c-176e234bbe61"}
{"level":"info","ts":"2026-09-18T09:48:57.327Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"4f5bf25c-61a8-4975-a00c-176e234bbe61","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:48:57.327Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"4f5bf25c-61a8-4975-a00c-176e234bbe61","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:00.149Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"fd088d55-f62e-43b9-a870-56ed55dcc07e"}
{"level":"info","ts":"2026-09-18T09:49:00.149Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"fd088d55-f62e-43b9-a870-56ed55dcc07e"}
{"level":"info","ts":"2026-09-18T09:49:00.149Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"fd088d55-f62e-43b9-a870-56ed55dcc07e"}
{"level":"info","ts":"2026-09-18T09:49:00.149Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"fd088d55-f62e-43b9-a870-56ed55dcc07e","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:00.149Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"fd088d55-f62e-43b9-a870-56ed55dcc07e","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:00.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"fd088d55-f62e-43b9-a870-56ed55dcc07e","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:00.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"fd088d55-f62e-43b9-a870-56ed55dcc07e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:00.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"fd088d55-f62e-43b9-a870-56ed55dcc07e","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:00.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"fd088d55-f62e-43b9-a870-56ed55dcc07e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:00.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"fd088d55-f62e-43b9-a870-56ed55dcc07e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:00.150Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"fd088d55-f62e-43b9-a870-56ed55dcc07e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:00.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"fd088d55-f62e-43b9-a870-56ed55dcc07e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:00.150Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"fd088d55-f62e-43b9-a870-56ed55dcc07e","podName":"counter-auth-6hjnc-small-group-worker-bhfr4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:47:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:49:00.150Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"fd088d55-f62e-43b9-a870-56ed55dcc07e","podName":"counter-auth-6hjnc-small-group-worker-bhfr4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:47:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:49:00.150Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"fd088d55-f62e-43b9-a870-56ed55dcc07e","name":"counter-auth-6hjnc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:49:00.161Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"3ca18634-d51e-44f8-8ce3-278edd53f307","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:00.162Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"3ca18634-d51e-44f8-8ce3-278edd53f307","pod":"counter-auth-6hjnc-head-wk2nd","ip":"10.132.0.33"}
{"level":"info","ts":"2026-09-18T09:49:00.162Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"3ca18634-d51e-44f8-8ce3-278edd53f307","pod":"counter-auth-6hjnc-small-group-worker-bhfr4","ip":"10.133.0.31"}
{"level":"info","ts":"2026-09-18T09:49:00.162Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"3ca18634-d51e-44f8-8ce3-278edd53f307","rayCluster":"counter-auth-6hjnc","podIPs":["10.132.0.33","10.133.0.31"]}
{"level":"info","ts":"2026-09-18T09:49:00.162Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"67cefe2c-b872-4b4a-8c78-8ac595253053","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:00.162Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"6b801969-3198-4e71-afd4-59888fcb909e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:00.162Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"6b801969-3198-4e71-afd4-59888fcb909e","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:00.162Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"6b801969-3198-4e71-afd4-59888fcb909e","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:00.162Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"6b801969-3198-4e71-afd4-59888fcb909e"}
{"level":"info","ts":"2026-09-18T09:49:00.162Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"6b801969-3198-4e71-afd4-59888fcb909e","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:49:00.162Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"6b801969-3198-4e71-afd4-59888fcb909e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:48:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:00.162Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"6b801969-3198-4e71-afd4-59888fcb909e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:49:00.172Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"67cefe2c-b872-4b4a-8c78-8ac595253053","name":"counter-auth-6hjnc-head"}
{"level":"info","ts":"2026-09-18T09:49:00.173Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"46e4e3d2-0053-415f-8fa8-516144bb296c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:00.173Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"46e4e3d2-0053-415f-8fa8-516144bb296c","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:00.173Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"46e4e3d2-0053-415f-8fa8-516144bb296c","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:00.174Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"46e4e3d2-0053-415f-8fa8-516144bb296c"}
{"level":"info","ts":"2026-09-18T09:49:00.174Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"46e4e3d2-0053-415f-8fa8-516144bb296c","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:49:00.174Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"46e4e3d2-0053-415f-8fa8-516144bb296c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:00.183Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"67cefe2c-b872-4b4a-8c78-8ac595253053","name":"counter-auth-6hjnc-workers"}
{"level":"info","ts":"2026-09-18T09:49:00.183Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"67cefe2c-b872-4b4a-8c78-8ac595253053","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:00.327Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d3bf4adb-b028-4449-9ed0-dea98b022d15","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:00.328Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d3bf4adb-b028-4449-9ed0-dea98b022d15","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:00.328Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d3bf4adb-b028-4449-9ed0-dea98b022d15","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:00.328Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d3bf4adb-b028-4449-9ed0-dea98b022d15"}
{"level":"info","ts":"2026-09-18T09:49:00.328Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d3bf4adb-b028-4449-9ed0-dea98b022d15","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:49:00.328Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"d3bf4adb-b028-4449-9ed0-dea98b022d15","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:00.594Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7ae8155b-94a9-456b-b077-49df6e9b309d","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:00.594Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7ae8155b-94a9-456b-b077-49df6e9b309d","pod":"counter-auth-6hjnc-head-wk2nd","ip":"10.132.0.33"}
{"level":"info","ts":"2026-09-18T09:49:00.594Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7ae8155b-94a9-456b-b077-49df6e9b309d","pod":"counter-auth-6hjnc-small-group-worker-bhfr4","ip":"10.133.0.31"}
{"level":"info","ts":"2026-09-18T09:49:00.594Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7ae8155b-94a9-456b-b077-49df6e9b309d","rayCluster":"counter-auth-6hjnc","podIPs":["10.132.0.33","10.133.0.31"]}
{"level":"info","ts":"2026-09-18T09:49:02.161Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"515ea1ae-f561-4c62-9b71-1aaa187d2ad2"}
{"level":"info","ts":"2026-09-18T09:49:02.161Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"515ea1ae-f561-4c62-9b71-1aaa187d2ad2"}
{"level":"info","ts":"2026-09-18T09:49:02.161Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"515ea1ae-f561-4c62-9b71-1aaa187d2ad2"}
{"level":"info","ts":"2026-09-18T09:49:02.161Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"515ea1ae-f561-4c62-9b71-1aaa187d2ad2","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:02.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"515ea1ae-f561-4c62-9b71-1aaa187d2ad2","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:02.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"515ea1ae-f561-4c62-9b71-1aaa187d2ad2","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:02.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"515ea1ae-f561-4c62-9b71-1aaa187d2ad2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:02.162Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"515ea1ae-f561-4c62-9b71-1aaa187d2ad2","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:02.162Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"515ea1ae-f561-4c62-9b71-1aaa187d2ad2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:02.162Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"515ea1ae-f561-4c62-9b71-1aaa187d2ad2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:02.162Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"515ea1ae-f561-4c62-9b71-1aaa187d2ad2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:02.162Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"515ea1ae-f561-4c62-9b71-1aaa187d2ad2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:02.162Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"515ea1ae-f561-4c62-9b71-1aaa187d2ad2","podName":"counter-auth-6hjnc-small-group-worker-bhfr4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:47:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:49:02.162Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"515ea1ae-f561-4c62-9b71-1aaa187d2ad2","podName":"counter-auth-6hjnc-small-group-worker-bhfr4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:47:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:49:02.162Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"515ea1ae-f561-4c62-9b71-1aaa187d2ad2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:02.162Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"515ea1ae-f561-4c62-9b71-1aaa187d2ad2","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:03.328Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0d5e1cb2-3fae-4a02-a900-cb363acc1d15","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:03.329Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0d5e1cb2-3fae-4a02-a900-cb363acc1d15","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:03.329Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0d5e1cb2-3fae-4a02-a900-cb363acc1d15","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:03.329Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0d5e1cb2-3fae-4a02-a900-cb363acc1d15"}
{"level":"info","ts":"2026-09-18T09:49:03.329Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0d5e1cb2-3fae-4a02-a900-cb363acc1d15","RayCluster":"counter-auth-6hjnc","State":""}
{"level":"info","ts":"2026-09-18T09:49:03.329Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"0d5e1cb2-3fae-4a02-a900-cb363acc1d15","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:04.943Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9dbdaca5-01d1-4d77-94a8-6ecdd69c94f3"}
{"level":"info","ts":"2026-09-18T09:49:04.943Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9dbdaca5-01d1-4d77-94a8-6ecdd69c94f3"}
{"level":"info","ts":"2026-09-18T09:49:04.943Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9dbdaca5-01d1-4d77-94a8-6ecdd69c94f3"}
{"level":"info","ts":"2026-09-18T09:49:04.943Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9dbdaca5-01d1-4d77-94a8-6ecdd69c94f3","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:04.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9dbdaca5-01d1-4d77-94a8-6ecdd69c94f3","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:04.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9dbdaca5-01d1-4d77-94a8-6ecdd69c94f3","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:04.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9dbdaca5-01d1-4d77-94a8-6ecdd69c94f3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:04.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9dbdaca5-01d1-4d77-94a8-6ecdd69c94f3","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:04.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9dbdaca5-01d1-4d77-94a8-6ecdd69c94f3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:04.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9dbdaca5-01d1-4d77-94a8-6ecdd69c94f3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:04.943Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9dbdaca5-01d1-4d77-94a8-6ecdd69c94f3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:04.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9dbdaca5-01d1-4d77-94a8-6ecdd69c94f3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:04.943Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"9dbdaca5-01d1-4d77-94a8-6ecdd69c94f3","name":"counter-auth-6hjnc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:04Z","stateTransitionTimes":{"ready":"2026-09-18T09:49:04Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:49:04Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:49:04.958Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7dfbc79c-e227-4d78-a731-e7183cb7837b","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:04.959Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bb528c00-df87-4ac1-8309-123ef65f53d5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:04.959Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bb528c00-df87-4ac1-8309-123ef65f53d5","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:04.959Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bb528c00-df87-4ac1-8309-123ef65f53d5","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:04.959Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"9e915be0-7e28-41ec-b304-afd2ca330434","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:04.959Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"9e915be0-7e28-41ec-b304-afd2ca330434","pod":"counter-auth-6hjnc-head-wk2nd","ip":"10.132.0.33"}
{"level":"info","ts":"2026-09-18T09:49:04.959Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"9e915be0-7e28-41ec-b304-afd2ca330434","pod":"counter-auth-6hjnc-small-group-worker-bhfr4","ip":"10.133.0.31"}
{"level":"info","ts":"2026-09-18T09:49:04.959Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bb528c00-df87-4ac1-8309-123ef65f53d5"}
{"level":"info","ts":"2026-09-18T09:49:04.959Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"9e915be0-7e28-41ec-b304-afd2ca330434","rayCluster":"counter-auth-6hjnc","podIPs":["10.132.0.33","10.133.0.31"]}
{"level":"info","ts":"2026-09-18T09:49:04.959Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bb528c00-df87-4ac1-8309-123ef65f53d5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","dashboardURL":"counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:04.959Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"bb528c00-df87-4ac1-8309-123ef65f53d5","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:49:04.985Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5f6d07ed-8506-469e-9c19-4c1918b95138","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:04.985Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5f6d07ed-8506-469e-9c19-4c1918b95138","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:04.985Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5f6d07ed-8506-469e-9c19-4c1918b95138","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:04.985Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5f6d07ed-8506-469e-9c19-4c1918b95138"}
{"level":"info","ts":"2026-09-18T09:49:04.985Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5f6d07ed-8506-469e-9c19-4c1918b95138","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:49:04.991Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7dfbc79c-e227-4d78-a731-e7183cb7837b","name":"counter-auth-6hjnc-head"}
{"level":"info","ts":"2026-09-18T09:49:05.005Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"5f6d07ed-8506-469e-9c19-4c1918b95138","JobId":"counter-auth-gxkp6"}
{"level":"info","ts":"2026-09-18T09:49:05.007Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7dfbc79c-e227-4d78-a731-e7183cb7837b","name":"counter-auth-6hjnc-workers"}
{"level":"info","ts":"2026-09-18T09:49:05.007Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7dfbc79c-e227-4d78-a731-e7183cb7837b","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:05.827Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"c5ea6854-4fe1-497b-85b2-f4c63181c539","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:05.827Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"c5ea6854-4fe1-497b-85b2-f4c63181c539","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:05.827Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"c5ea6854-4fe1-497b-85b2-f4c63181c539","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:05.827Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"c5ea6854-4fe1-497b-85b2-f4c63181c539"}
{"level":"info","ts":"2026-09-18T09:49:05.827Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"c5ea6854-4fe1-497b-85b2-f4c63181c539","rayCluster":"counter-auth-6hjnc","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:49:05.830Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"c5ea6854-4fe1-497b-85b2-f4c63181c539","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","dashboardURL":"counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:49:05Z"},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","dashboardURL":"counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:04Z","stateTransitionTimes":{"ready":"2026-09-18T09:49:04Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:49:04Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:05.830Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"c5ea6854-4fe1-497b-85b2-f4c63181c539","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:49:05.844Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b036e3bb-f948-40e0-ac7e-6a3dff61c2a0","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:05.844Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b036e3bb-f948-40e0-ac7e-6a3dff61c2a0","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:05.844Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b036e3bb-f948-40e0-ac7e-6a3dff61c2a0","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:05.844Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b036e3bb-f948-40e0-ac7e-6a3dff61c2a0"}
{"level":"info","ts":"2026-09-18T09:49:05.844Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b036e3bb-f948-40e0-ac7e-6a3dff61c2a0","rayCluster":"counter-auth-6hjnc","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:49:05.847Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b036e3bb-f948-40e0-ac7e-6a3dff61c2a0","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:49:05Z"},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","dashboardURL":"counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:04Z","stateTransitionTimes":{"ready":"2026-09-18T09:49:04Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:49:04Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:49:05Z"},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","dashboardURL":"counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:04Z","stateTransitionTimes":{"ready":"2026-09-18T09:49:04Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:49:04Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:06.330Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"90476a7e-0905-407b-8cb6-0dba13acebba","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:06.330Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"90476a7e-0905-407b-8cb6-0dba13acebba","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:06.330Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"90476a7e-0905-407b-8cb6-0dba13acebba","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:06.330Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"90476a7e-0905-407b-8cb6-0dba13acebba"}
{"level":"info","ts":"2026-09-18T09:49:06.330Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"90476a7e-0905-407b-8cb6-0dba13acebba","rayCluster":"counter-auth-6hjnc","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:49:06.333Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"90476a7e-0905-407b-8cb6-0dba13acebba","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:49:05Z"},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","dashboardURL":"counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:04Z","stateTransitionTimes":{"ready":"2026-09-18T09:49:04Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:49:04Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:49:05Z"},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","dashboardURL":"counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:04Z","stateTransitionTimes":{"ready":"2026-09-18T09:49:04Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:49:04Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:06.959Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"13c97c8f-b444-4c79-a2a6-2a691916d03e"}
{"level":"info","ts":"2026-09-18T09:49:06.959Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"13c97c8f-b444-4c79-a2a6-2a691916d03e"}
{"level":"info","ts":"2026-09-18T09:49:06.959Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"13c97c8f-b444-4c79-a2a6-2a691916d03e"}
{"level":"info","ts":"2026-09-18T09:49:06.959Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"13c97c8f-b444-4c79-a2a6-2a691916d03e","1 head service found":"counter-auth-6hjnc-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:06.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"13c97c8f-b444-4c79-a2a6-2a691916d03e","Found 1 head Pod":"counter-auth-6hjnc-head-wk2nd","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:06.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"13c97c8f-b444-4c79-a2a6-2a691916d03e","head Pod":"counter-auth-6hjnc-head-wk2nd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-6hjnc-head-wk2nd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:06.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"13c97c8f-b444-4c79-a2a6-2a691916d03e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:06.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"13c97c8f-b444-4c79-a2a6-2a691916d03e","worker Pod":"counter-auth-6hjnc-small-group-worker-bhfr4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-6hjnc-small-group-worker-bhfr4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:06.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"13c97c8f-b444-4c79-a2a6-2a691916d03e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:06.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"13c97c8f-b444-4c79-a2a6-2a691916d03e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:06.960Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"13c97c8f-b444-4c79-a2a6-2a691916d03e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:06.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"13c97c8f-b444-4c79-a2a6-2a691916d03e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:06.960Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"13c97c8f-b444-4c79-a2a6-2a691916d03e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:06.960Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"13c97c8f-b444-4c79-a2a6-2a691916d03e","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:09.334Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b7c3a421-c66e-4978-b496-00304bb541f8","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:09.334Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b7c3a421-c66e-4978-b496-00304bb541f8","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:09.334Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b7c3a421-c66e-4978-b496-00304bb541f8","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:09.334Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b7c3a421-c66e-4978-b496-00304bb541f8"}
{"level":"info","ts":"2026-09-18T09:49:09.334Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b7c3a421-c66e-4978-b496-00304bb541f8","rayCluster":"counter-auth-6hjnc","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:49:09.338Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b7c3a421-c66e-4978-b496-00304bb541f8","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:49:05Z"},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","dashboardURL":"counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:04Z","stateTransitionTimes":{"ready":"2026-09-18T09:49:04Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:49:04Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:49:05Z"},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","dashboardURL":"counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:04Z","stateTransitionTimes":{"ready":"2026-09-18T09:49:04Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:49:04Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:09.338Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b7c3a421-c66e-4978-b496-00304bb541f8","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:49:09.359Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b2bb0cc9-6af5-4419-9fd3-309c11a96017","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:09.359Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b2bb0cc9-6af5-4419-9fd3-309c11a96017","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:09.359Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b2bb0cc9-6af5-4419-9fd3-309c11a96017","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:09.359Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b2bb0cc9-6af5-4419-9fd3-309c11a96017"}
{"level":"info","ts":"2026-09-18T09:49:09.360Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b2bb0cc9-6af5-4419-9fd3-309c11a96017","rayCluster":"counter-auth-6hjnc","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:49:09.363Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"b2bb0cc9-6af5-4419-9fd3-309c11a96017","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:49:05Z"},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","dashboardURL":"counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:04Z","stateTransitionTimes":{"ready":"2026-09-18T09:49:04Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:49:04Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:49:05Z"},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","dashboardURL":"counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:04Z","stateTransitionTimes":{"ready":"2026-09-18T09:49:04Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:49:04Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:12.360Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"df493d4d-3801-472d-afa7-3a4bb66c4072","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:12.360Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"df493d4d-3801-472d-afa7-3a4bb66c4072","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:12.361Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"df493d4d-3801-472d-afa7-3a4bb66c4072","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:12.361Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"df493d4d-3801-472d-afa7-3a4bb66c4072"}
{"level":"info","ts":"2026-09-18T09:49:12.361Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"df493d4d-3801-472d-afa7-3a4bb66c4072","rayCluster":"counter-auth-6hjnc","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:49:12.364Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"df493d4d-3801-472d-afa7-3a4bb66c4072","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:49:05Z"},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","dashboardURL":"counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T09:47:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:04Z","stateTransitionTimes":{"ready":"2026-09-18T09:49:04Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:49:04Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:49:05Z","endTime":"2026-09-18T09:49:09Z"},"jobId":"counter-auth-gxkp6","rayClusterName":"counter-auth-6hjnc","dashboardURL":"counter-auth-6hjnc-head-svc.test-ns-q2lj2.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-09-18T09:47:30Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:04Z","stateTransitionTimes":{"ready":"2026-09-18T09:49:04Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"counter-auth-6hjnc-head-wk2nd","serviceName":"counter-auth-6hjnc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:49:04Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:47:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:47:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:12.364Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"df493d4d-3801-472d-afa7-3a4bb66c4072","old JobStatus":"RUNNING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-09-18T09:49:12.383Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"efaa6161-9f75-43df-bcaa-2b17388bc089","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:12.383Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"efaa6161-9f75-43df-bcaa-2b17388bc089","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:49:12 +0000 UTC","Now":"2026-09-18T09:49:12.383Z","ShutdownTime":"2026-09-18T09:49:12.000Z"}
{"level":"info","ts":"2026-09-18T09:49:12.403Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"efaa6161-9f75-43df-bcaa-2b17388bc089","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:12.403Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"efaa6161-9f75-43df-bcaa-2b17388bc089","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T09:49:12.403Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"efaa6161-9f75-43df-bcaa-2b17388bc089","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:12.404Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"e8dd3c1c-70e3-4f00-998e-0749355a0062","namespacedName":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:12.404Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"e8dd3c1c-70e3-4f00-998e-0749355a0062","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:12.404Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"e8dd3c1c-70e3-4f00-998e-0749355a0062","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:12.404Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"e8dd3c1c-70e3-4f00-998e-0749355a0062","namespace":"test-ns-q2lj2","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-09-18T09:49:12.404Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"e8dd3c1c-70e3-4f00-998e-0749355a0062","name":"kuberay-gateway-access","namespace":"test-ns-q2lj2","remainingClusters":1}
{"level":"info","ts":"2026-09-18T09:49:12.404Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"1bcbe770-d8f4-4700-a172-ed04be660cdc","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:12.404Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"1bcbe770-d8f4-4700-a172-ed04be660cdc","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:49:12 +0000 UTC","Now":"2026-09-18T09:49:12.404Z","ShutdownTime":"2026-09-18T09:49:12.000Z"}
{"level":"info","ts":"2026-09-18T09:49:12.404Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"1bcbe770-d8f4-4700-a172-ed04be660cdc","RayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:12.404Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"1bcbe770-d8f4-4700-a172-ed04be660cdc","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T09:49:12.404Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"1bcbe770-d8f4-4700-a172-ed04be660cdc","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:12.405Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"reconcileID":"bdca2802-7a64-4899-8009-cdfe410e0dcc"}
{"level":"info","ts":"2026-09-18T09:49:12.405Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"f7421ba0-8ab9-46ab-b1dc-8a123a956ee6"}
{"level":"info","ts":"2026-09-18T09:49:12.405Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"2789af15-4f4f-4005-9f66-98cc071591c9","rayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:12.405Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"2789af15-4f4f-4005-9f66-98cc071591c9","pod":"counter-auth-6hjnc-head-wk2nd","ip":"10.132.0.33"}
{"level":"info","ts":"2026-09-18T09:49:12.405Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"2789af15-4f4f-4005-9f66-98cc071591c9","pod":"counter-auth-6hjnc-small-group-worker-bhfr4","ip":"10.133.0.31"}
{"level":"info","ts":"2026-09-18T09:49:12.405Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"2789af15-4f4f-4005-9f66-98cc071591c9","rayCluster":"counter-auth-6hjnc","podIPs":["10.132.0.33","10.133.0.31"]}
{"level":"info","ts":"2026-09-18T09:49:12.416Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"e8dd3c1c-70e3-4f00-998e-0749355a0062","name":"test-ns-q2lj2-counter-auth-6hjnc","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T09:49:12.430Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"e8dd3c1c-70e3-4f00-998e-0749355a0062","configMap":"kube-rbac-proxy-config-counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:12.438Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"e8dd3c1c-70e3-4f00-998e-0749355a0062","serviceAccount":"counter-auth-6hjnc-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:12.481Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:12.481Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:49:12.496Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"e8dd3c1c-70e3-4f00-998e-0749355a0062","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:12.496Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"192684f0-c76d-4638-b434-9d235d49d784","namespacedName":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:12.496Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"192684f0-c76d-4638-b434-9d235d49d784","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:12.496Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"192684f0-c76d-4638-b434-9d235d49d784","cluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:12.496Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"192684f0-c76d-4638-b434-9d235d49d784","namespace":"test-ns-q2lj2","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-09-18T09:49:12.496Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"192684f0-c76d-4638-b434-9d235d49d784","name":"kuberay-gateway-access","namespace":"test-ns-q2lj2","remainingClusters":1}
{"level":"info","ts":"2026-09-18T09:49:12.497Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"dba48209-b129-4651-a2f8-e0631b768cd2","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:12.497Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"dba48209-b129-4651-a2f8-e0631b768cd2","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:49:12 +0000 UTC","Now":"2026-09-18T09:49:12.497Z","ShutdownTime":"2026-09-18T09:49:12.000Z"}
{"level":"info","ts":"2026-09-18T09:49:12.497Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"dba48209-b129-4651-a2f8-e0631b768cd2","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:12.497Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"d2181300-bc73-4d02-be6f-dabe0e573dcc","namespace":"test-ns-q2lj2","clusterName":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:12.497Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"bded7cfa-e167-4551-9752-3b7899ca6ddc"}
{"level":"info","ts":"2026-09-18T09:49:12.497Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"dba48209-b129-4651-a2f8-e0631b768cd2","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T09:49:12.497Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"dba48209-b129-4651-a2f8-e0631b768cd2","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-6hjnc"}
{"level":"error","ts":"2026-09-18T09:49:12.503Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"192684f0-c76d-4638-b434-9d235d49d784","error":"rayclusters.ray.io \"counter-auth-6hjnc\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:196\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:137\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:12.503Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"192684f0-c76d-4638-b434-9d235d49d784"}
{"level":"error","ts":"2026-09-18T09:49:12.503Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"192684f0-c76d-4638-b434-9d235d49d784","error":"rayclusters.ray.io \"counter-auth-6hjnc\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:12.503Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"f1f9aad6-9f03-4272-8c11-9539a17a5597","namespacedName":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:12.503Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"f1f9aad6-9f03-4272-8c11-9539a17a5597"}
{"level":"info","ts":"2026-09-18T09:49:12.503Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"f1f9aad6-9f03-4272-8c11-9539a17a5597","namespace":"test-ns-q2lj2","authClusters":1}
{"level":"info","ts":"2026-09-18T09:49:12.503Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"f1f9aad6-9f03-4272-8c11-9539a17a5597","namespace":"test-ns-q2lj2","authClusters":1}
{"level":"info","ts":"2026-09-18T09:49:12.508Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"13c3ccf5-2995-4451-87e1-d56899e767a5","namespacedName":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:12.508Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"13c3ccf5-2995-4451-87e1-d56899e767a5"}
{"level":"info","ts":"2026-09-18T09:49:12.508Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"13c3ccf5-2995-4451-87e1-d56899e767a5","namespace":"test-ns-q2lj2","authClusters":1}
{"level":"info","ts":"2026-09-18T09:49:12.508Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"13c3ccf5-2995-4451-87e1-d56899e767a5","namespace":"test-ns-q2lj2","authClusters":1}
{"level":"info","ts":"2026-09-18T09:49:12.513Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"47706ba2-0717-4c94-9e7d-9cb9b9d43e7c","namespacedName":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:12.514Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"47706ba2-0717-4c94-9e7d-9cb9b9d43e7c"}
{"level":"info","ts":"2026-09-18T09:49:12.514Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"47706ba2-0717-4c94-9e7d-9cb9b9d43e7c","namespace":"test-ns-q2lj2","authClusters":1}
{"level":"info","ts":"2026-09-18T09:49:12.514Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"47706ba2-0717-4c94-9e7d-9cb9b9d43e7c","namespace":"test-ns-q2lj2","authClusters":1}
{"level":"info","ts":"2026-09-18T09:49:12.514Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"868e2f42-4103-4599-bbca-71666a1c5a3e"}
{"level":"info","ts":"2026-09-18T09:49:12.517Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"7ed0df64-7fa2-43a6-bf22-15814298ff71"}
{"level":"error","ts":"2026-09-18T09:49:12.517Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"d2181300-bc73-4d02-be6f-dabe0e573dcc","issuer":"ray-selfsigned-issuer-counter-auth-6hjnc","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-auth-6hjnc\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:49:12.541Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"d2181300-bc73-4d02-be6f-dabe0e573dcc","certificate":"ray-worker-cert-counter-auth-6hjnc","error":"certificates.cert-manager.io \"ray-worker-cert-counter-auth-6hjnc\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:49:12.555Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"d2181300-bc73-4d02-be6f-dabe0e573dcc","certificate":"ray-ca-certificate-counter-auth-6hjnc","error":"certificates.cert-manager.io \"ray-ca-certificate-counter-auth-6hjnc\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:12.614Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"d2181300-bc73-4d02-be6f-dabe0e573dcc","errorCount":3}
{"level":"info","ts":"2026-09-18T09:49:12.614Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"d2181300-bc73-4d02-be6f-dabe0e573dcc"}
{"level":"error","ts":"2026-09-18T09:49:12.614Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"d2181300-bc73-4d02-be6f-dabe0e573dcc","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-auth-6hjnc\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:12.619Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"6b11bc3a-d34b-4cb8-93fe-ea3e1d198506","namespace":"test-ns-q2lj2","clusterName":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:12.619Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"counter-auth-6hjnc","reconcileID":"6b11bc3a-d34b-4cb8-93fe-ea3e1d198506","namespace":"test-ns-q2lj2","clusterName":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:13.098Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e2caa8e8-5ce1-4d28-ab7a-0b3badcb146a","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:13.098Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e2caa8e8-5ce1-4d28-ab7a-0b3badcb146a","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:49:13.111Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e2caa8e8-5ce1-4d28-ab7a-0b3badcb146a"}
{"level":"info","ts":"2026-09-18T09:49:13.112Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e2caa8e8-5ce1-4d28-ab7a-0b3badcb146a","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:49:13.112Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e2caa8e8-5ce1-4d28-ab7a-0b3badcb146a","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:49:13.112Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e2caa8e8-5ce1-4d28-ab7a-0b3badcb146a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:49:13.167Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"1bda3dad-cc0b-4045-aa8e-dd4125f686dd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:13.167Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"1bda3dad-cc0b-4045-aa8e-dd4125f686dd","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.167Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"1bda3dad-cc0b-4045-aa8e-dd4125f686dd","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:13.167Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"1bda3dad-cc0b-4045-aa8e-dd4125f686dd","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:13.173Z","logger":"raycluster-resource","msg":"default","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.173Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-2qcbf","namespace":"test-ns-chvpt"}
{"level":"info","ts":"2026-09-18T09:49:13.187Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57cb8926-2e3e-4477-8785-0ec651040246"}
{"level":"info","ts":"2026-09-18T09:49:13.187Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57cb8926-2e3e-4477-8785-0ec651040246"}
{"level":"info","ts":"2026-09-18T09:49:13.187Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57cb8926-2e3e-4477-8785-0ec651040246"}
{"level":"info","ts":"2026-09-18T09:49:13.188Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"8725f5a7-dcfe-43a6-ac12-ea802a376b9e","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.188Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"8725f5a7-dcfe-43a6-ac12-ea802a376b9e","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.188Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"e35f826a-7f5a-4a61-992b-4363cced632e","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.188Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"e3a21531-a766-4c4e-9f87-fe3678fc11bf","namespacedName":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:13.188Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"e3a21531-a766-4c4e-9f87-fe3678fc11bf","mode":"IntegratedOAuth","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.188Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"e3a21531-a766-4c4e-9f87-fe3678fc11bf","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.188Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"e3a21531-a766-4c4e-9f87-fe3678fc11bf","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.189Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"1bda3dad-cc0b-4045-aa8e-dd4125f686dd","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:13.189Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"1bda3dad-cc0b-4045-aa8e-dd4125f686dd"}
{"level":"info","ts":"2026-09-18T09:49:13.189Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"1bda3dad-cc0b-4045-aa8e-dd4125f686dd","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:13.189Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"1bda3dad-cc0b-4045-aa8e-dd4125f686dd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:49:13.189Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4d095790-3a40-4c30-bc09-0b3d989bb969","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:13.189Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4d095790-3a40-4c30-bc09-0b3d989bb969","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.189Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4d095790-3a40-4c30-bc09-0b3d989bb969","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:13.266Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4d095790-3a40-4c30-bc09-0b3d989bb969"}
{"level":"info","ts":"2026-09-18T09:49:13.266Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4d095790-3a40-4c30-bc09-0b3d989bb969","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:13.266Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57cb8926-2e3e-4477-8785-0ec651040246","name":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:13.266Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4d095790-3a40-4c30-bc09-0b3d989bb969","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:49:13.266Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57cb8926-2e3e-4477-8785-0ec651040246","error":"head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"error","ts":"2026-09-18T09:49:13.266Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57cb8926-2e3e-4477-8785-0ec651040246","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.266Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57cb8926-2e3e-4477-8785-0ec651040246","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"info","ts":"2026-09-18T09:49:13.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57cb8926-2e3e-4477-8785-0ec651040246","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57cb8926-2e3e-4477-8785-0ec651040246","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57cb8926-2e3e-4477-8785-0ec651040246","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.266Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57cb8926-2e3e-4477-8785-0ec651040246","name":"long-running-2qcbf","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:49:13.267Z","logger":"raycluster-resource","msg":"default","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.267Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-2qcbf","namespace":"test-ns-chvpt"}
{"level":"info","ts":"2026-09-18T09:49:13.267Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"8725f5a7-dcfe-43a6-ac12-ea802a376b9e","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.267Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"8725f5a7-dcfe-43a6-ac12-ea802a376b9e","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.275Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"e35f826a-7f5a-4a61-992b-4363cced632e","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.275Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"017ed35c-dc7b-4014-b8f5-246840acaaae","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.276Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"10caa40c-3e04-40c7-8a40-b25689c3b232","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:13.276Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"10caa40c-3e04-40c7-8a40-b25689c3b232","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.276Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"10caa40c-3e04-40c7-8a40-b25689c3b232","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:13.277Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"10caa40c-3e04-40c7-8a40-b25689c3b232"}
{"level":"info","ts":"2026-09-18T09:49:13.277Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"10caa40c-3e04-40c7-8a40-b25689c3b232","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:13.277Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"10caa40c-3e04-40c7-8a40-b25689c3b232","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:49:13.280Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"8725f5a7-dcfe-43a6-ac12-ea802a376b9e","rayCluster":"long-running-2qcbf","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T09:49:13.280Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"8725f5a7-dcfe-43a6-ac12-ea802a376b9e","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.282Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57cb8926-2e3e-4477-8785-0ec651040246","name":"long-running-2qcbf","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-2qcbf\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-chvpt","name":"long-running-2qcbf"}}
{"level":"info","ts":"2026-09-18T09:49:13.282Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57cb8926-2e3e-4477-8785-0ec651040246"}
{"level":"error","ts":"2026-09-18T09:49:13.282Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57cb8926-2e3e-4477-8785-0ec651040246","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.282Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"255b1831-8f47-49c3-9eec-eaaddb994693"}
{"level":"info","ts":"2026-09-18T09:49:13.282Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"255b1831-8f47-49c3-9eec-eaaddb994693"}
{"level":"info","ts":"2026-09-18T09:49:13.282Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"255b1831-8f47-49c3-9eec-eaaddb994693"}
{"level":"info","ts":"2026-09-18T09:49:13.282Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"255b1831-8f47-49c3-9eec-eaaddb994693","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:13.282Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"255b1831-8f47-49c3-9eec-eaaddb994693","error":"head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"error","ts":"2026-09-18T09:49:13.282Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"255b1831-8f47-49c3-9eec-eaaddb994693","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.282Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"255b1831-8f47-49c3-9eec-eaaddb994693","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"info","ts":"2026-09-18T09:49:13.282Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"255b1831-8f47-49c3-9eec-eaaddb994693","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.282Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"255b1831-8f47-49c3-9eec-eaaddb994693","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.282Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"255b1831-8f47-49c3-9eec-eaaddb994693","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.282Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"255b1831-8f47-49c3-9eec-eaaddb994693","name":"long-running-2qcbf","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:49:13.366Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"e3a21531-a766-4c4e-9f87-fe3678fc11bf","name":"long-running-2qcbf-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:13.366Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"8725f5a7-dcfe-43a6-ac12-ea802a376b9e","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.367Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"255b1831-8f47-49c3-9eec-eaaddb994693"}
{"level":"error","ts":"2026-09-18T09:49:13.367Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"255b1831-8f47-49c3-9eec-eaaddb994693","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.367Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"60138b65-892b-4b14-aeed-4a19e9c48e3e"}
{"level":"info","ts":"2026-09-18T09:49:13.367Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"60138b65-892b-4b14-aeed-4a19e9c48e3e"}
{"level":"info","ts":"2026-09-18T09:49:13.367Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"60138b65-892b-4b14-aeed-4a19e9c48e3e"}
{"level":"info","ts":"2026-09-18T09:49:13.367Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"60138b65-892b-4b14-aeed-4a19e9c48e3e","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:13.367Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"60138b65-892b-4b14-aeed-4a19e9c48e3e","error":"head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"error","ts":"2026-09-18T09:49:13.367Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"60138b65-892b-4b14-aeed-4a19e9c48e3e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.367Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"60138b65-892b-4b14-aeed-4a19e9c48e3e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"info","ts":"2026-09-18T09:49:13.367Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"60138b65-892b-4b14-aeed-4a19e9c48e3e","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.367Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"60138b65-892b-4b14-aeed-4a19e9c48e3e","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.367Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"60138b65-892b-4b14-aeed-4a19e9c48e3e","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.367Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"60138b65-892b-4b14-aeed-4a19e9c48e3e","name":"long-running-2qcbf","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:49:13.368Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bccbf50d-bb15-4382-8e2b-fa2fa5936148","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:13.368Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bccbf50d-bb15-4382-8e2b-fa2fa5936148","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.368Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bccbf50d-bb15-4382-8e2b-fa2fa5936148","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:13.368Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bccbf50d-bb15-4382-8e2b-fa2fa5936148"}
{"level":"info","ts":"2026-09-18T09:49:13.368Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bccbf50d-bb15-4382-8e2b-fa2fa5936148","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:13.368Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bccbf50d-bb15-4382-8e2b-fa2fa5936148","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:13.368Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bccbf50d-bb15-4382-8e2b-fa2fa5936148","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:49:13.373Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"e3a21531-a766-4c4e-9f87-fe3678fc11bf","name":"kuberay-gateway-access","namespace":"test-ns-chvpt","operation":"created"}
{"level":"info","ts":"2026-09-18T09:49:13.373Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"017ed35c-dc7b-4014-b8f5-246840acaaae","name":"long-running-2qcbf-head"}
{"level":"info","ts":"2026-09-18T09:49:13.378Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"60138b65-892b-4b14-aeed-4a19e9c48e3e","name":"long-running-2qcbf","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-2qcbf\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-chvpt","name":"long-running-2qcbf"}}
{"level":"info","ts":"2026-09-18T09:49:13.378Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"60138b65-892b-4b14-aeed-4a19e9c48e3e"}
{"level":"error","ts":"2026-09-18T09:49:13.378Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"60138b65-892b-4b14-aeed-4a19e9c48e3e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.378Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"53945b10-e042-4f5e-b4c8-5671a1d8b06e"}
{"level":"info","ts":"2026-09-18T09:49:13.378Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"53945b10-e042-4f5e-b4c8-5671a1d8b06e"}
{"level":"info","ts":"2026-09-18T09:49:13.378Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"53945b10-e042-4f5e-b4c8-5671a1d8b06e"}
{"level":"info","ts":"2026-09-18T09:49:13.378Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"53945b10-e042-4f5e-b4c8-5671a1d8b06e","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:13.378Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"53945b10-e042-4f5e-b4c8-5671a1d8b06e","error":"head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"error","ts":"2026-09-18T09:49:13.378Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"53945b10-e042-4f5e-b4c8-5671a1d8b06e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.378Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"53945b10-e042-4f5e-b4c8-5671a1d8b06e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"info","ts":"2026-09-18T09:49:13.378Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"53945b10-e042-4f5e-b4c8-5671a1d8b06e","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.378Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"53945b10-e042-4f5e-b4c8-5671a1d8b06e","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.378Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"53945b10-e042-4f5e-b4c8-5671a1d8b06e","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.378Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"53945b10-e042-4f5e-b4c8-5671a1d8b06e"}
{"level":"error","ts":"2026-09-18T09:49:13.378Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"53945b10-e042-4f5e-b4c8-5671a1d8b06e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.384Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"f85f9bb3-d0b5-4bf3-904f-eb26a9e74ac7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:13.384Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"f85f9bb3-d0b5-4bf3-904f-eb26a9e74ac7","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.384Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"f85f9bb3-d0b5-4bf3-904f-eb26a9e74ac7","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:13.384Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"f85f9bb3-d0b5-4bf3-904f-eb26a9e74ac7"}
{"level":"info","ts":"2026-09-18T09:49:13.384Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"f85f9bb3-d0b5-4bf3-904f-eb26a9e74ac7","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:13.384Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"f85f9bb3-d0b5-4bf3-904f-eb26a9e74ac7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:13.386Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"8725f5a7-dcfe-43a6-ac12-ea802a376b9e","rayCluster":"long-running-2qcbf","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:49:13.466Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf2a84b2-e85d-4510-af9a-8f70ca4eee76"}
{"level":"info","ts":"2026-09-18T09:49:13.466Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf2a84b2-e85d-4510-af9a-8f70ca4eee76"}
{"level":"info","ts":"2026-09-18T09:49:13.466Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf2a84b2-e85d-4510-af9a-8f70ca4eee76"}
{"level":"info","ts":"2026-09-18T09:49:13.466Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"e3a21531-a766-4c4e-9f87-fe3678fc11bf","name":"test-ns-chvpt-long-running-2qcbf","operation":"created"}
{"level":"info","ts":"2026-09-18T09:49:13.466Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"017ed35c-dc7b-4014-b8f5-246840acaaae","name":"long-running-2qcbf-workers"}
{"level":"info","ts":"2026-09-18T09:49:13.466Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"017ed35c-dc7b-4014-b8f5-246840acaaae","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.466Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"c86586ed-8bb3-4221-936c-7d40aed6fa70","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.466Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf2a84b2-e85d-4510-af9a-8f70ca4eee76","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:13.466Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf2a84b2-e85d-4510-af9a-8f70ca4eee76","error":"head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"error","ts":"2026-09-18T09:49:13.466Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf2a84b2-e85d-4510-af9a-8f70ca4eee76","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.466Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf2a84b2-e85d-4510-af9a-8f70ca4eee76","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"info","ts":"2026-09-18T09:49:13.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf2a84b2-e85d-4510-af9a-8f70ca4eee76","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf2a84b2-e85d-4510-af9a-8f70ca4eee76","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf2a84b2-e85d-4510-af9a-8f70ca4eee76","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.467Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf2a84b2-e85d-4510-af9a-8f70ca4eee76"}
{"level":"error","ts":"2026-09-18T09:49:13.467Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf2a84b2-e85d-4510-af9a-8f70ca4eee76","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.467Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"8725f5a7-dcfe-43a6-ac12-ea802a376b9e","rayCluster":"long-running-2qcbf","podIPs":[]}
{"level":"error","ts":"2026-09-18T09:49:13.467Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"8725f5a7-dcfe-43a6-ac12-ea802a376b9e","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-2qcbf\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.467Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"8725f5a7-dcfe-43a6-ac12-ea802a376b9e"}
{"level":"error","ts":"2026-09-18T09:49:13.467Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"8725f5a7-dcfe-43a6-ac12-ea802a376b9e","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-2qcbf\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.467Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"7bcfefea-a565-40e9-bf37-4740eb360d0b","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.473Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"e3a21531-a766-4c4e-9f87-fe3678fc11bf","name":"kube-rbac-proxy-config-long-running-2qcbf","operation":"created"}
{"level":"error","ts":"2026-09-18T09:49:13.481Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"e3a21531-a766-4c4e-9f87-fe3678fc11bf","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-2qcbf\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.481Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"e3a21531-a766-4c4e-9f87-fe3678fc11bf"}
{"level":"error","ts":"2026-09-18T09:49:13.481Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"e3a21531-a766-4c4e-9f87-fe3678fc11bf","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-2qcbf\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.481Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"38813ba7-a504-4053-b0a1-bd495806871a","namespacedName":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:13.481Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"38813ba7-a504-4053-b0a1-bd495806871a","mode":"IntegratedOAuth","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.481Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"38813ba7-a504-4053-b0a1-bd495806871a","cluster":"long-running-2qcbf"}
{"level":"error","ts":"2026-09-18T09:49:13.483Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"7bcfefea-a565-40e9-bf37-4740eb360d0b","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-2qcbf\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:49:13.483Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"7bcfefea-a565-40e9-bf37-4740eb360d0b","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-2qcbf\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.483Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"7bcfefea-a565-40e9-bf37-4740eb360d0b"}
{"level":"error","ts":"2026-09-18T09:49:13.483Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"7bcfefea-a565-40e9-bf37-4740eb360d0b","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-2qcbf\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.483Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"ff29295e-e5be-4c3f-96b9-29e6174052ef","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.483Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"ff29295e-e5be-4c3f-96b9-29e6174052ef"}
{"level":"info","ts":"2026-09-18T09:49:13.484Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"c86586ed-8bb3-4221-936c-7d40aed6fa70","name":"long-running-2qcbf-head"}
{"level":"info","ts":"2026-09-18T09:49:13.494Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"b6ff5d1a-5855-4eea-aacf-9ca78f619dac","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.494Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"b6ff5d1a-5855-4eea-aacf-9ca78f619dac"}
{"level":"info","ts":"2026-09-18T09:49:13.495Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"38813ba7-a504-4053-b0a1-bd495806871a","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.495Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"6b1f83c0-5581-470e-9405-7f371aa73cbd","namespacedName":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:13.495Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"6b1f83c0-5581-470e-9405-7f371aa73cbd","mode":"IntegratedOAuth","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.495Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"6b1f83c0-5581-470e-9405-7f371aa73cbd","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.495Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0acecf8a-d679-4032-914a-1334ecb937c0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:13.495Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0acecf8a-d679-4032-914a-1334ecb937c0","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.495Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0acecf8a-d679-4032-914a-1334ecb937c0","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:13.495Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0acecf8a-d679-4032-914a-1334ecb937c0"}
{"level":"info","ts":"2026-09-18T09:49:13.495Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0acecf8a-d679-4032-914a-1334ecb937c0","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:13.495Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0acecf8a-d679-4032-914a-1334ecb937c0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:13.495Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0acecf8a-d679-4032-914a-1334ecb937c0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:49:13.566Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"07e5d183-554d-49f2-b6fc-8a53dc9086a6"}
{"level":"info","ts":"2026-09-18T09:49:13.566Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"07e5d183-554d-49f2-b6fc-8a53dc9086a6"}
{"level":"info","ts":"2026-09-18T09:49:13.566Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"07e5d183-554d-49f2-b6fc-8a53dc9086a6"}
{"level":"info","ts":"2026-09-18T09:49:13.566Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"07e5d183-554d-49f2-b6fc-8a53dc9086a6","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:13.566Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"07e5d183-554d-49f2-b6fc-8a53dc9086a6","error":"head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"error","ts":"2026-09-18T09:49:13.566Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"07e5d183-554d-49f2-b6fc-8a53dc9086a6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.566Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"07e5d183-554d-49f2-b6fc-8a53dc9086a6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"info","ts":"2026-09-18T09:49:13.566Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"07e5d183-554d-49f2-b6fc-8a53dc9086a6","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.566Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"07e5d183-554d-49f2-b6fc-8a53dc9086a6","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.566Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"07e5d183-554d-49f2-b6fc-8a53dc9086a6","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.566Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"07e5d183-554d-49f2-b6fc-8a53dc9086a6"}
{"level":"error","ts":"2026-09-18T09:49:13.566Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"07e5d183-554d-49f2-b6fc-8a53dc9086a6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.566Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"70302a1e-4766-4279-98cb-152f069c9f04","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.566Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"70302a1e-4766-4279-98cb-152f069c9f04"}
{"level":"info","ts":"2026-09-18T09:49:13.566Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"c86586ed-8bb3-4221-936c-7d40aed6fa70","name":"long-running-2qcbf-workers"}
{"level":"info","ts":"2026-09-18T09:49:13.566Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"c86586ed-8bb3-4221-936c-7d40aed6fa70","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.566Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"88c20d1c-13dc-4111-b091-cac3d7d462c1","cluster":"long-running-2qcbf"}
{"level":"error","ts":"2026-09-18T09:49:13.573Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"6b1f83c0-5581-470e-9405-7f371aa73cbd","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-2qcbf\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.574Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"6b1f83c0-5581-470e-9405-7f371aa73cbd"}
{"level":"error","ts":"2026-09-18T09:49:13.574Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"6b1f83c0-5581-470e-9405-7f371aa73cbd","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-2qcbf\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.579Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"259e9353-0f3e-4502-b9f9-1ef568bd6ee6","namespacedName":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:13.579Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"259e9353-0f3e-4502-b9f9-1ef568bd6ee6","mode":"IntegratedOAuth","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.579Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"259e9353-0f3e-4502-b9f9-1ef568bd6ee6","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.579Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"88c20d1c-13dc-4111-b091-cac3d7d462c1","name":"long-running-2qcbf-head"}
{"level":"info","ts":"2026-09-18T09:49:13.582Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"5a93b5c9-6b1a-4da6-8c14-5af27a10baff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:13.582Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"5a93b5c9-6b1a-4da6-8c14-5af27a10baff","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.582Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"5a93b5c9-6b1a-4da6-8c14-5af27a10baff","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:13.582Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"5a93b5c9-6b1a-4da6-8c14-5af27a10baff"}
{"level":"info","ts":"2026-09-18T09:49:13.582Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"5a93b5c9-6b1a-4da6-8c14-5af27a10baff","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:13.582Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"5a93b5c9-6b1a-4da6-8c14-5af27a10baff","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:13.591Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"259e9353-0f3e-4502-b9f9-1ef568bd6ee6","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.592Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"88c20d1c-13dc-4111-b091-cac3d7d462c1","name":"long-running-2qcbf-workers"}
{"level":"info","ts":"2026-09-18T09:49:13.592Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"88c20d1c-13dc-4111-b091-cac3d7d462c1","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:13.726Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"13ffaf9f-12eb-4aa8-91aa-754ded5ed8f2"}
{"level":"info","ts":"2026-09-18T09:49:13.726Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"13ffaf9f-12eb-4aa8-91aa-754ded5ed8f2"}
{"level":"info","ts":"2026-09-18T09:49:13.726Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"13ffaf9f-12eb-4aa8-91aa-754ded5ed8f2"}
{"level":"info","ts":"2026-09-18T09:49:13.726Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"13ffaf9f-12eb-4aa8-91aa-754ded5ed8f2","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:13.726Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"13ffaf9f-12eb-4aa8-91aa-754ded5ed8f2","error":"head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"error","ts":"2026-09-18T09:49:13.726Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"13ffaf9f-12eb-4aa8-91aa-754ded5ed8f2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:13.726Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"13ffaf9f-12eb-4aa8-91aa-754ded5ed8f2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"info","ts":"2026-09-18T09:49:13.727Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"13ffaf9f-12eb-4aa8-91aa-754ded5ed8f2","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.727Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"13ffaf9f-12eb-4aa8-91aa-754ded5ed8f2","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.727Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"13ffaf9f-12eb-4aa8-91aa-754ded5ed8f2","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:13.727Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"13ffaf9f-12eb-4aa8-91aa-754ded5ed8f2"}
{"level":"error","ts":"2026-09-18T09:49:13.727Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"13ffaf9f-12eb-4aa8-91aa-754ded5ed8f2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:14.047Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"b40aa6c6-6be0-40f5-9795-9923833c05e5"}
{"level":"info","ts":"2026-09-18T09:49:14.047Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"b40aa6c6-6be0-40f5-9795-9923833c05e5"}
{"level":"info","ts":"2026-09-18T09:49:14.047Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"b40aa6c6-6be0-40f5-9795-9923833c05e5"}
{"level":"info","ts":"2026-09-18T09:49:14.047Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"b40aa6c6-6be0-40f5-9795-9923833c05e5","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:14.047Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"b40aa6c6-6be0-40f5-9795-9923833c05e5","error":"head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"error","ts":"2026-09-18T09:49:14.047Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"b40aa6c6-6be0-40f5-9795-9923833c05e5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:14.047Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"b40aa6c6-6be0-40f5-9795-9923833c05e5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"info","ts":"2026-09-18T09:49:14.047Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"b40aa6c6-6be0-40f5-9795-9923833c05e5","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:14.047Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"b40aa6c6-6be0-40f5-9795-9923833c05e5","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:14.047Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"b40aa6c6-6be0-40f5-9795-9923833c05e5","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:14.048Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"b40aa6c6-6be0-40f5-9795-9923833c05e5"}
{"level":"error","ts":"2026-09-18T09:49:14.048Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"b40aa6c6-6be0-40f5-9795-9923833c05e5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:14.688Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40641e53-f12a-4a82-94c7-3f367f8c9179"}
{"level":"info","ts":"2026-09-18T09:49:14.688Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40641e53-f12a-4a82-94c7-3f367f8c9179"}
{"level":"info","ts":"2026-09-18T09:49:14.688Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40641e53-f12a-4a82-94c7-3f367f8c9179"}
{"level":"info","ts":"2026-09-18T09:49:14.688Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40641e53-f12a-4a82-94c7-3f367f8c9179","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:14.688Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40641e53-f12a-4a82-94c7-3f367f8c9179","error":"head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"error","ts":"2026-09-18T09:49:14.688Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40641e53-f12a-4a82-94c7-3f367f8c9179","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:14.688Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40641e53-f12a-4a82-94c7-3f367f8c9179","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"info","ts":"2026-09-18T09:49:14.689Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40641e53-f12a-4a82-94c7-3f367f8c9179","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:14.689Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40641e53-f12a-4a82-94c7-3f367f8c9179","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:14.689Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40641e53-f12a-4a82-94c7-3f367f8c9179","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:14.689Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40641e53-f12a-4a82-94c7-3f367f8c9179"}
{"level":"error","ts":"2026-09-18T09:49:14.689Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40641e53-f12a-4a82-94c7-3f367f8c9179","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:15.384Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"88c0bf00-a4cf-443c-8dca-ab65a307cfe4","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:15.384Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"88c0bf00-a4cf-443c-8dca-ab65a307cfe4","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:49:12 +0000 UTC","Now":"2026-09-18T09:49:15.384Z","ShutdownTime":"2026-09-18T09:49:12.000Z"}
{"level":"info","ts":"2026-09-18T09:49:15.384Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"88c0bf00-a4cf-443c-8dca-ab65a307cfe4","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:15.384Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"88c0bf00-a4cf-443c-8dca-ab65a307cfe4","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T09:49:15.384Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"88c0bf00-a4cf-443c-8dca-ab65a307cfe4","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:15.969Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05f7090c-6f20-4b0f-91f2-1695cf2b3fa2"}
{"level":"info","ts":"2026-09-18T09:49:15.969Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05f7090c-6f20-4b0f-91f2-1695cf2b3fa2"}
{"level":"info","ts":"2026-09-18T09:49:15.969Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05f7090c-6f20-4b0f-91f2-1695cf2b3fa2"}
{"level":"info","ts":"2026-09-18T09:49:15.969Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05f7090c-6f20-4b0f-91f2-1695cf2b3fa2","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:15.969Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05f7090c-6f20-4b0f-91f2-1695cf2b3fa2","error":"head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"error","ts":"2026-09-18T09:49:15.969Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05f7090c-6f20-4b0f-91f2-1695cf2b3fa2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:15.969Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05f7090c-6f20-4b0f-91f2-1695cf2b3fa2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found"}
{"level":"info","ts":"2026-09-18T09:49:15.969Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05f7090c-6f20-4b0f-91f2-1695cf2b3fa2","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:15.969Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05f7090c-6f20-4b0f-91f2-1695cf2b3fa2","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:15.970Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05f7090c-6f20-4b0f-91f2-1695cf2b3fa2","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-2qcbf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:15.970Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05f7090c-6f20-4b0f-91f2-1695cf2b3fa2"}
{"level":"error","ts":"2026-09-18T09:49:15.970Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05f7090c-6f20-4b0f-91f2-1695cf2b3fa2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-2qcbf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:16.168Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"077439d1-d14c-407c-9ddd-8bcacb040cb8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:16.168Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"077439d1-d14c-407c-9ddd-8bcacb040cb8","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:16.168Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"077439d1-d14c-407c-9ddd-8bcacb040cb8","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:16.168Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"077439d1-d14c-407c-9ddd-8bcacb040cb8"}
{"level":"info","ts":"2026-09-18T09:49:16.168Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"077439d1-d14c-407c-9ddd-8bcacb040cb8","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:16.168Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"077439d1-d14c-407c-9ddd-8bcacb040cb8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:18.079Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"6f996559-d258-40fe-a235-227fc8e4f8a2"}
{"level":"info","ts":"2026-09-18T09:49:18.079Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"6f996559-d258-40fe-a235-227fc8e4f8a2"}
{"level":"info","ts":"2026-09-18T09:49:18.079Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"6f996559-d258-40fe-a235-227fc8e4f8a2"}
{"level":"info","ts":"2026-09-18T09:49:18.079Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"6f996559-d258-40fe-a235-227fc8e4f8a2","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:18.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"6f996559-d258-40fe-a235-227fc8e4f8a2","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:18.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"6f996559-d258-40fe-a235-227fc8e4f8a2","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:18.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"6f996559-d258-40fe-a235-227fc8e4f8a2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:18.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"6f996559-d258-40fe-a235-227fc8e4f8a2","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:18.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"6f996559-d258-40fe-a235-227fc8e4f8a2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:18.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"6f996559-d258-40fe-a235-227fc8e4f8a2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:18.079Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"6f996559-d258-40fe-a235-227fc8e4f8a2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:18.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"6f996559-d258-40fe-a235-227fc8e4f8a2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:18.079Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"6f996559-d258-40fe-a235-227fc8e4f8a2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:18.079Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"6f996559-d258-40fe-a235-227fc8e4f8a2","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:18.085Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"38077e88-8850-4b85-ae14-f98e148da4c5"}
{"level":"info","ts":"2026-09-18T09:49:18.085Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"38077e88-8850-4b85-ae14-f98e148da4c5"}
{"level":"info","ts":"2026-09-18T09:49:18.085Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"38077e88-8850-4b85-ae14-f98e148da4c5"}
{"level":"info","ts":"2026-09-18T09:49:18.085Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"38077e88-8850-4b85-ae14-f98e148da4c5","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:18.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"38077e88-8850-4b85-ae14-f98e148da4c5","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:18.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"38077e88-8850-4b85-ae14-f98e148da4c5","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:18.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"38077e88-8850-4b85-ae14-f98e148da4c5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:18.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"38077e88-8850-4b85-ae14-f98e148da4c5","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:18.086Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"38077e88-8850-4b85-ae14-f98e148da4c5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:18.086Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"38077e88-8850-4b85-ae14-f98e148da4c5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:18.086Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"38077e88-8850-4b85-ae14-f98e148da4c5","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:18.086Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"38077e88-8850-4b85-ae14-f98e148da4c5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:18.086Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"38077e88-8850-4b85-ae14-f98e148da4c5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:18.086Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"38077e88-8850-4b85-ae14-f98e148da4c5","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:18.101Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"91a238ab-cc61-4c38-ae00-1483b099ab3b"}
{"level":"info","ts":"2026-09-18T09:49:18.101Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"91a238ab-cc61-4c38-ae00-1483b099ab3b"}
{"level":"info","ts":"2026-09-18T09:49:18.102Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"91a238ab-cc61-4c38-ae00-1483b099ab3b"}
{"level":"info","ts":"2026-09-18T09:49:18.102Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"91a238ab-cc61-4c38-ae00-1483b099ab3b","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:18.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"91a238ab-cc61-4c38-ae00-1483b099ab3b","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:18.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"91a238ab-cc61-4c38-ae00-1483b099ab3b","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:18.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"91a238ab-cc61-4c38-ae00-1483b099ab3b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:18.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"91a238ab-cc61-4c38-ae00-1483b099ab3b","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:18.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"91a238ab-cc61-4c38-ae00-1483b099ab3b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:18.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"91a238ab-cc61-4c38-ae00-1483b099ab3b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:18.102Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"91a238ab-cc61-4c38-ae00-1483b099ab3b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:18.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"91a238ab-cc61-4c38-ae00-1483b099ab3b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:18.102Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"91a238ab-cc61-4c38-ae00-1483b099ab3b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:18.102Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"91a238ab-cc61-4c38-ae00-1483b099ab3b","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:18.105Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"2328ef38-94d7-4033-9cb6-52d1bae767d7"}
{"level":"info","ts":"2026-09-18T09:49:18.106Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"2328ef38-94d7-4033-9cb6-52d1bae767d7"}
{"level":"info","ts":"2026-09-18T09:49:18.106Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"2328ef38-94d7-4033-9cb6-52d1bae767d7"}
{"level":"info","ts":"2026-09-18T09:49:18.106Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"2328ef38-94d7-4033-9cb6-52d1bae767d7","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:18.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"2328ef38-94d7-4033-9cb6-52d1bae767d7","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:18.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"2328ef38-94d7-4033-9cb6-52d1bae767d7","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:18.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"2328ef38-94d7-4033-9cb6-52d1bae767d7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:18.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"2328ef38-94d7-4033-9cb6-52d1bae767d7","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-bcffx-small-group-worker-dcw8j. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:18.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"2328ef38-94d7-4033-9cb6-52d1bae767d7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:18.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"2328ef38-94d7-4033-9cb6-52d1bae767d7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:18.106Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"2328ef38-94d7-4033-9cb6-52d1bae767d7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:18.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"2328ef38-94d7-4033-9cb6-52d1bae767d7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:18.106Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"2328ef38-94d7-4033-9cb6-52d1bae767d7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:18.106Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"2328ef38-94d7-4033-9cb6-52d1bae767d7","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:18.531Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa"}
{"level":"info","ts":"2026-09-18T09:49:18.531Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa"}
{"level":"info","ts":"2026-09-18T09:49:18.531Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa"}
{"level":"info","ts":"2026-09-18T09:49:18.531Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:18.531Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa"}
{"level":"info","ts":"2026-09-18T09:49:18.531Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:18.531Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa"}
{"level":"info","ts":"2026-09-18T09:49:18.531Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","mode":"IntegratedOAuth","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:18.531Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","cluster":"long-running-2qcbf","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:18.531Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","cluster":"long-running-2qcbf","authType":"IntegratedOAuth","serviceAccount":"long-running-2qcbf-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:49:18.531Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-2qcbf","ray.io/group":"headgroup","ray.io/identifier":"long-running-2qcbf-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:49:18.531Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:49:18.531Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:18.531Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:18.531Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","enabled":true}
{"level":"info","ts":"2026-09-18T09:49:18.553Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","name":"long-running-2qcbf-head-cz5d7"}
{"level":"info","ts":"2026-09-18T09:49:18.553Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:18.553Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:18.553Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:49:18.553Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:49:18.553Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:18.553Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:49:18.553Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa"}
{"level":"info","ts":"2026-09-18T09:49:18.553Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","nodeType":"worker","rayStartParams":{"address":"long-running-2qcbf-head-svc.test-ns-chvpt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:49:18.553Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","rayStartCmd":"ray start  --address=long-running-2qcbf-head-svc.test-ns-chvpt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:49:18.553Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-2qcbf-head-svc.test-ns-chvpt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:49:18.553Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","enabled":true}
{"level":"info","ts":"2026-09-18T09:49:18.568Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","name":"long-running-2qcbf-small-group-worker-94s9h"}
{"level":"info","ts":"2026-09-18T09:49:18.569Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:18.569Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:18.569Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"acd503e0-a96d-4cef-ae1a-936ef44b2eaa","name":"long-running-2qcbf","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"7615f6bb-1b8f-48b3-8e76-8273d4e0e963","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"eff40d77-2de7-43c8-b084-4dff4f71c21d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"eff40d77-2de7-43c8-b084-4dff4f71c21d","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"eff40d77-2de7-43c8-b084-4dff4f71c21d","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"e8bd8772-339e-432f-92e1-6e057061a77b","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"eff40d77-2de7-43c8-b084-4dff4f71c21d"}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"eff40d77-2de7-43c8-b084-4dff4f71c21d","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"e8bd8772-339e-432f-92e1-6e057061a77b","rayCluster":"long-running-2qcbf","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"eff40d77-2de7-43c8-b084-4dff4f71c21d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf44e320-4430-4458-93c8-80ecb25f3de8"}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"eff40d77-2de7-43c8-b084-4dff4f71c21d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf44e320-4430-4458-93c8-80ecb25f3de8"}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf44e320-4430-4458-93c8-80ecb25f3de8"}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf44e320-4430-4458-93c8-80ecb25f3de8","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf44e320-4430-4458-93c8-80ecb25f3de8","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf44e320-4430-4458-93c8-80ecb25f3de8","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf44e320-4430-4458-93c8-80ecb25f3de8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf44e320-4430-4458-93c8-80ecb25f3de8","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf44e320-4430-4458-93c8-80ecb25f3de8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf44e320-4430-4458-93c8-80ecb25f3de8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf44e320-4430-4458-93c8-80ecb25f3de8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf44e320-4430-4458-93c8-80ecb25f3de8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf44e320-4430-4458-93c8-80ecb25f3de8","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:18.582Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf44e320-4430-4458-93c8-80ecb25f3de8","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:18.583Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf44e320-4430-4458-93c8-80ecb25f3de8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:18.583Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"cf44e320-4430-4458-93c8-80ecb25f3de8","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:18.596Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"7615f6bb-1b8f-48b3-8e76-8273d4e0e963","name":"long-running-2qcbf-head"}
{"level":"info","ts":"2026-09-18T09:49:18.596Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f17f6a9c-1c4e-4e97-8233-f435de087df5"}
{"level":"info","ts":"2026-09-18T09:49:18.596Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f17f6a9c-1c4e-4e97-8233-f435de087df5"}
{"level":"info","ts":"2026-09-18T09:49:18.596Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f17f6a9c-1c4e-4e97-8233-f435de087df5"}
{"level":"info","ts":"2026-09-18T09:49:18.596Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f17f6a9c-1c4e-4e97-8233-f435de087df5","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:18.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f17f6a9c-1c4e-4e97-8233-f435de087df5","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:18.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f17f6a9c-1c4e-4e97-8233-f435de087df5","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:18.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f17f6a9c-1c4e-4e97-8233-f435de087df5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:18.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f17f6a9c-1c4e-4e97-8233-f435de087df5","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:18.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f17f6a9c-1c4e-4e97-8233-f435de087df5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:18.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f17f6a9c-1c4e-4e97-8233-f435de087df5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:18.666Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f17f6a9c-1c4e-4e97-8233-f435de087df5","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:18.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f17f6a9c-1c4e-4e97-8233-f435de087df5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:18.666Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f17f6a9c-1c4e-4e97-8233-f435de087df5","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:18.666Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f17f6a9c-1c4e-4e97-8233-f435de087df5","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:18.666Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f17f6a9c-1c4e-4e97-8233-f435de087df5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:18.666Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f17f6a9c-1c4e-4e97-8233-f435de087df5","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"df412e1a-2e66-4a1a-8ec4-a418dfb0c806","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"df412e1a-2e66-4a1a-8ec4-a418dfb0c806","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"df412e1a-2e66-4a1a-8ec4-a418dfb0c806","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"df412e1a-2e66-4a1a-8ec4-a418dfb0c806"}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"df412e1a-2e66-4a1a-8ec4-a418dfb0c806","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6ab7f013-cda9-49f3-a623-4e4338718356"}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6ab7f013-cda9-49f3-a623-4e4338718356"}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6ab7f013-cda9-49f3-a623-4e4338718356"}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"df412e1a-2e66-4a1a-8ec4-a418dfb0c806","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6ab7f013-cda9-49f3-a623-4e4338718356","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6ab7f013-cda9-49f3-a623-4e4338718356","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6ab7f013-cda9-49f3-a623-4e4338718356","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6ab7f013-cda9-49f3-a623-4e4338718356","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6ab7f013-cda9-49f3-a623-4e4338718356","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6ab7f013-cda9-49f3-a623-4e4338718356","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6ab7f013-cda9-49f3-a623-4e4338718356","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6ab7f013-cda9-49f3-a623-4e4338718356","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:18.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6ab7f013-cda9-49f3-a623-4e4338718356","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:18.668Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6ab7f013-cda9-49f3-a623-4e4338718356","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:18.668Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6ab7f013-cda9-49f3-a623-4e4338718356","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:18.668Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6ab7f013-cda9-49f3-a623-4e4338718356","name":"long-running-2qcbf","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:49:18.683Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"7615f6bb-1b8f-48b3-8e76-8273d4e0e963","name":"long-running-2qcbf-workers"}
{"level":"info","ts":"2026-09-18T09:49:18.683Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"7615f6bb-1b8f-48b3-8e76-8273d4e0e963","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:18.684Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"aee661ec-dde8-4a23-aec4-701913ee21ad","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:18.684Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a5b1cc2d-c7e2-457b-b4e8-5ef2f3afed22","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:18.684Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a5b1cc2d-c7e2-457b-b4e8-5ef2f3afed22","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:18.684Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a5b1cc2d-c7e2-457b-b4e8-5ef2f3afed22","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:18.684Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"6dadc5e7-c923-4b27-881d-d93e481ef834","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:18.684Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a5b1cc2d-c7e2-457b-b4e8-5ef2f3afed22"}
{"level":"info","ts":"2026-09-18T09:49:18.684Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a5b1cc2d-c7e2-457b-b4e8-5ef2f3afed22","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:18.684Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a5b1cc2d-c7e2-457b-b4e8-5ef2f3afed22","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:18.684Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a5b1cc2d-c7e2-457b-b4e8-5ef2f3afed22","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:49:18.684Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"aee661ec-dde8-4a23-aec4-701913ee21ad","rayCluster":"long-running-2qcbf","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:49:18.685Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"003b80d6-0db2-4046-beb8-c56960cfab77"}
{"level":"info","ts":"2026-09-18T09:49:18.685Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"003b80d6-0db2-4046-beb8-c56960cfab77"}
{"level":"info","ts":"2026-09-18T09:49:18.685Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"003b80d6-0db2-4046-beb8-c56960cfab77"}
{"level":"info","ts":"2026-09-18T09:49:18.685Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"003b80d6-0db2-4046-beb8-c56960cfab77","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:18.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"003b80d6-0db2-4046-beb8-c56960cfab77","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:18.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"003b80d6-0db2-4046-beb8-c56960cfab77","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-bcffx-head-s6qj5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:18.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"003b80d6-0db2-4046-beb8-c56960cfab77","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:18.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"003b80d6-0db2-4046-beb8-c56960cfab77","worker Pod":"fail-bcffx-small-group-worker-dcw8j","shouldDelete":true,"reason":"The worker Pod fail-bcffx-small-group-worker-dcw8j status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-09-18T09:49:18.688Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"003b80d6-0db2-4046-beb8-c56960cfab77","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:18.688Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"003b80d6-0db2-4046-beb8-c56960cfab77","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-09-18T09:49:18.689Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"003b80d6-0db2-4046-beb8-c56960cfab77","name":"fail-bcffx","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:46:30Z","stateTransitionTimes":{"ready":"2026-09-18T09:46:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:46:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","stateTransitionTimes":{"ready":"2026-09-18T09:46:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:46:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:49:18.766Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"6dadc5e7-c923-4b27-881d-d93e481ef834","name":"long-running-2qcbf-head"}
{"level":"info","ts":"2026-09-18T09:49:18.766Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"003b80d6-0db2-4046-beb8-c56960cfab77"}
{"level":"info","ts":"2026-09-18T09:49:18.766Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f69422ab-2915-4bac-8bb9-73ca1729f9e4","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"error","ts":"2026-09-18T09:49:18.766Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"003b80d6-0db2-4046-beb8-c56960cfab77","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:18.766Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"149c1fc6-b1c0-4adc-8c20-a5d1bc47d74d","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:18.766Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"25dafbd1-47ed-4eaa-8885-2d35ebbb93b9"}
{"level":"info","ts":"2026-09-18T09:49:18.766Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"25dafbd1-47ed-4eaa-8885-2d35ebbb93b9"}
{"level":"info","ts":"2026-09-18T09:49:18.766Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"25dafbd1-47ed-4eaa-8885-2d35ebbb93b9"}
{"level":"info","ts":"2026-09-18T09:49:18.766Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"25dafbd1-47ed-4eaa-8885-2d35ebbb93b9","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:18.766Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"149c1fc6-b1c0-4adc-8c20-a5d1bc47d74d","pod":"fail-bcffx-head-s6qj5","ip":"10.133.0.29"}
{"level":"info","ts":"2026-09-18T09:49:18.766Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"149c1fc6-b1c0-4adc-8c20-a5d1bc47d74d","pod":"fail-bcffx-small-group-worker-dcw8j","ip":"10.132.0.31"}
{"level":"info","ts":"2026-09-18T09:49:18.767Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"149c1fc6-b1c0-4adc-8c20-a5d1bc47d74d","rayCluster":"fail-bcffx","podIPs":["10.133.0.29","10.132.0.31"]}
{"level":"info","ts":"2026-09-18T09:49:18.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"25dafbd1-47ed-4eaa-8885-2d35ebbb93b9","Found 1 head Pod":"fail-bcffx-head-s6qj5","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-09-18 09:46:09 +0000 UTC,FinishedAt:2026-09-18 09:49:18 +0000 UTC,ContainerID:cri-o://c122f2cfd7a346c2ba8b3ee65e5cde40678d12160512783fca2e51f369a743fd,}"}
{"level":"info","ts":"2026-09-18T09:49:18.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"25dafbd1-47ed-4eaa-8885-2d35ebbb93b9","head Pod":"fail-bcffx-head-s6qj5","shouldDelete":true,"reason":"The head Pod fail-bcffx-head-s6qj5 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-09-18T09:49:18.767Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"80862523-605b-4f82-a14c-0c8f618905bf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:18.767Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"80862523-605b-4f82-a14c-0c8f618905bf","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:18.768Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"80862523-605b-4f82-a14c-0c8f618905bf","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:18.768Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"80862523-605b-4f82-a14c-0c8f618905bf"}
{"level":"info","ts":"2026-09-18T09:49:18.768Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"80862523-605b-4f82-a14c-0c8f618905bf","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:18.768Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"80862523-605b-4f82-a14c-0c8f618905bf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"error","ts":"2026-09-18T09:49:18.775Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"25dafbd1-47ed-4eaa-8885-2d35ebbb93b9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod fail-bcffx-head-s6qj5 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:18.775Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"25dafbd1-47ed-4eaa-8885-2d35ebbb93b9","error":"The head Pod fail-bcffx-head-s6qj5 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-09-18T09:49:18.776Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"25dafbd1-47ed-4eaa-8885-2d35ebbb93b9","name":"fail-bcffx","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","stateTransitionTimes":{"ready":"2026-09-18T09:46:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:46:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:46:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","stateTransitionTimes":{"ready":"2026-09-18T09:46:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:18Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:46:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:49:18.785Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"6dadc5e7-c923-4b27-881d-d93e481ef834","name":"long-running-2qcbf-workers"}
{"level":"info","ts":"2026-09-18T09:49:18.785Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"6dadc5e7-c923-4b27-881d-d93e481ef834","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:18.785Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"0de51f4b-ce01-4451-bd79-b0f08e04a9f5","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:18.791Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"0de51f4b-ce01-4451-bd79-b0f08e04a9f5","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:18.794Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"62027c02-2491-442e-8526-3edb90344a52","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:18.794Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"5d44c11a-09cd-4a88-a649-9e5911396be0","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:18.794Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"25dafbd1-47ed-4eaa-8885-2d35ebbb93b9"}
{"level":"error","ts":"2026-09-18T09:49:18.794Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"25dafbd1-47ed-4eaa-8885-2d35ebbb93b9","error":"The head Pod fail-bcffx-head-s6qj5 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:18.794Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5afadc55-6e22-440b-a0c1-7800138dbdc5"}
{"level":"info","ts":"2026-09-18T09:49:18.794Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5afadc55-6e22-440b-a0c1-7800138dbdc5"}
{"level":"info","ts":"2026-09-18T09:49:18.794Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"5d44c11a-09cd-4a88-a649-9e5911396be0","pod":"fail-bcffx-head-s6qj5","ip":"10.133.0.29"}
{"level":"info","ts":"2026-09-18T09:49:18.794Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5afadc55-6e22-440b-a0c1-7800138dbdc5"}
{"level":"info","ts":"2026-09-18T09:49:18.794Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5afadc55-6e22-440b-a0c1-7800138dbdc5","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:18.794Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"869de9ba-7451-422a-a23f-52e075378433","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:18.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5afadc55-6e22-440b-a0c1-7800138dbdc5","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-09-18T09:49:18.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5afadc55-6e22-440b-a0c1-7800138dbdc5","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T09:49:18.795Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5afadc55-6e22-440b-a0c1-7800138dbdc5","podName":"fail-bcffx-small-group-worker-dcw8j","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-09-18T09:49:18.794Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"5d44c11a-09cd-4a88-a649-9e5911396be0","pod":"fail-bcffx-small-group-worker-dcw8j","ip":"10.132.0.31"}
{"level":"info","ts":"2026-09-18T09:49:18.795Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5afadc55-6e22-440b-a0c1-7800138dbdc5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:18.795Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"5afadc55-6e22-440b-a0c1-7800138dbdc5","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:18.795Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"5d44c11a-09cd-4a88-a649-9e5911396be0","rayCluster":"fail-bcffx","podIPs":["10.133.0.29","10.132.0.31"]}
{"level":"info","ts":"2026-09-18T09:49:18.803Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"869de9ba-7451-422a-a23f-52e075378433","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:18.805Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8f5d0d8c-265f-44e3-a9cc-c72a59d48f2f"}
{"level":"info","ts":"2026-09-18T09:49:18.805Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8f5d0d8c-265f-44e3-a9cc-c72a59d48f2f"}
{"level":"info","ts":"2026-09-18T09:49:18.805Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8f5d0d8c-265f-44e3-a9cc-c72a59d48f2f"}
{"level":"info","ts":"2026-09-18T09:49:18.805Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8f5d0d8c-265f-44e3-a9cc-c72a59d48f2f","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:18.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8f5d0d8c-265f-44e3-a9cc-c72a59d48f2f","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-09-18T09:49:18.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8f5d0d8c-265f-44e3-a9cc-c72a59d48f2f","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T09:49:18.805Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8f5d0d8c-265f-44e3-a9cc-c72a59d48f2f","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-09-18T09:49:18.805Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8f5d0d8c-265f-44e3-a9cc-c72a59d48f2f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:18.805Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"8f5d0d8c-265f-44e3-a9cc-c72a59d48f2f","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:19.023Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a09369d8-176b-4bb6-86b9-85cc9c44c3a1"}
{"level":"info","ts":"2026-09-18T09:49:19.023Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a09369d8-176b-4bb6-86b9-85cc9c44c3a1"}
{"level":"info","ts":"2026-09-18T09:49:19.023Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a09369d8-176b-4bb6-86b9-85cc9c44c3a1"}
{"level":"info","ts":"2026-09-18T09:49:19.023Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a09369d8-176b-4bb6-86b9-85cc9c44c3a1","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:19.023Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a09369d8-176b-4bb6-86b9-85cc9c44c3a1","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-09-18T09:49:19.023Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a09369d8-176b-4bb6-86b9-85cc9c44c3a1","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T09:49:19.023Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a09369d8-176b-4bb6-86b9-85cc9c44c3a1","podName":"fail-bcffx-head-s6qj5","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-09-18T09:49:19.023Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a09369d8-176b-4bb6-86b9-85cc9c44c3a1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:19.023Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a09369d8-176b-4bb6-86b9-85cc9c44c3a1","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:19.029Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17"}
{"level":"info","ts":"2026-09-18T09:49:19.029Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17"}
{"level":"info","ts":"2026-09-18T09:49:19.029Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17"}
{"level":"info","ts":"2026-09-18T09:49:19.029Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:19.029Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17"}
{"level":"info","ts":"2026-09-18T09:49:19.029Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.029Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17"}
{"level":"info","ts":"2026-09-18T09:49:19.029Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:19.029Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","cluster":"fail-bcffx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.029Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","cluster":"fail-bcffx","authType":"IntegratedOAuth","serviceAccount":"fail-bcffx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:49:19.029Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-bcffx","ray.io/group":"headgroup","ray.io/identifier":"fail-bcffx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:49:19.029Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:49:19.029Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.029Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.029Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","enabled":true}
{"level":"error","ts":"2026-09-18T09:49:19.034Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.034Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:49:19.034Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.034Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.034Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.034Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","name":"fail-bcffx","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","stateTransitionTimes":{"ready":"2026-09-18T09:46:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"fail-bcffx-head-s6qj5","serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:18Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:46:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:19Z","stateTransitionTimes":{"ready":"2026-09-18T09:46:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-bcffx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:18Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:46:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:45:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:45:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T09:49:19Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:49:19.049Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17"}
{"level":"error","ts":"2026-09-18T09:49:19.049Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"a1aae7f3-8cfa-4e14-9474-a4b827472d17","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.049Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"9972559e-5607-47bc-9066-b9df06959708","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:19.049Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"9972559e-5607-47bc-9066-b9df06959708","pod":"fail-bcffx-small-group-worker-dcw8j","ip":"10.132.0.31"}
{"level":"info","ts":"2026-09-18T09:49:19.049Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"a0e555c2-5114-4e02-8c34-c633443ed3af","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:19.049Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"46db5fdb-1b1b-4544-a935-88c6de4a973c","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:19.055Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39"}
{"level":"info","ts":"2026-09-18T09:49:19.055Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39"}
{"level":"info","ts":"2026-09-18T09:49:19.055Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39"}
{"level":"info","ts":"2026-09-18T09:49:19.055Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:19.055Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39"}
{"level":"info","ts":"2026-09-18T09:49:19.055Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.055Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39"}
{"level":"info","ts":"2026-09-18T09:49:19.055Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:19.055Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39","cluster":"fail-bcffx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.055Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39","cluster":"fail-bcffx","authType":"IntegratedOAuth","serviceAccount":"fail-bcffx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:49:19.055Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-bcffx","ray.io/group":"headgroup","ray.io/identifier":"fail-bcffx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:49:19.055Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:49:19.055Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.055Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.055Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39","enabled":true}
{"level":"info","ts":"2026-09-18T09:49:19.055Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"46db5fdb-1b1b-4544-a935-88c6de4a973c","cluster":"fail-bcffx"}
{"level":"error","ts":"2026-09-18T09:49:19.058Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.058Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:49:19.058Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.058Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.058Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.058Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39"}
{"level":"error","ts":"2026-09-18T09:49:19.058Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"f631fe80-6232-4c80-b5ff-23bb80e84b39","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.061Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"9972559e-5607-47bc-9066-b9df06959708","certificate":"ray-head-cert-fail-bcffx","dnsNames":["fail-bcffx-head-svc","localhost","fail-bcffx-head-svc.test-ns-q2lj2.svc","fail-bcffx-head-svc.test-ns-q2lj2.svc.cluster.local"],"ipAddresses":["10.132.0.31","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:49:19.069Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a"}
{"level":"info","ts":"2026-09-18T09:49:19.069Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a"}
{"level":"info","ts":"2026-09-18T09:49:19.069Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a"}
{"level":"info","ts":"2026-09-18T09:49:19.069Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:19.069Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a"}
{"level":"info","ts":"2026-09-18T09:49:19.069Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.069Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a"}
{"level":"info","ts":"2026-09-18T09:49:19.069Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:19.069Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a","cluster":"fail-bcffx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.069Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a","cluster":"fail-bcffx","authType":"IntegratedOAuth","serviceAccount":"fail-bcffx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:49:19.069Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-bcffx","ray.io/group":"headgroup","ray.io/identifier":"fail-bcffx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:49:19.069Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:49:19.070Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.070Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.070Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a","enabled":true}
{"level":"error","ts":"2026-09-18T09:49:19.075Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.075Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:49:19.075Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.075Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.075Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.075Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a"}
{"level":"error","ts":"2026-09-18T09:49:19.075Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"06ced03e-bec9-45f7-8972-b3a049887a6a","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.077Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"9972559e-5607-47bc-9066-b9df06959708","certificate":"ray-worker-cert-fail-bcffx","dnsNames":["fail-bcffx-worker-svc","localhost","fail-bcffx-small-group","fail-bcffx-worker-svc.test-ns-q2lj2.svc","fail-bcffx-worker-svc.test-ns-q2lj2.svc.cluster.local","*.fail-bcffx-worker-svc.test-ns-q2lj2.svc","*.fail-bcffx-worker-svc.test-ns-q2lj2.svc.cluster.local","*-worker-*.test-ns-q2lj2.svc","*-worker-*.test-ns-q2lj2.svc.cluster.local"],"ipAddresses":["10.132.0.31","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:49:19.077Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"9972559e-5607-47bc-9066-b9df06959708"}
{"level":"info","ts":"2026-09-18T09:49:19.096Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917"}
{"level":"info","ts":"2026-09-18T09:49:19.096Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917"}
{"level":"info","ts":"2026-09-18T09:49:19.096Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917"}
{"level":"info","ts":"2026-09-18T09:49:19.096Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:19.096Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917"}
{"level":"info","ts":"2026-09-18T09:49:19.096Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.096Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917"}
{"level":"info","ts":"2026-09-18T09:49:19.096Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:19.097Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917","cluster":"fail-bcffx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.097Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917","cluster":"fail-bcffx","authType":"IntegratedOAuth","serviceAccount":"fail-bcffx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:49:19.097Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-bcffx","ray.io/group":"headgroup","ray.io/identifier":"fail-bcffx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:49:19.097Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:49:19.097Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.097Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.097Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917","enabled":true}
{"level":"error","ts":"2026-09-18T09:49:19.100Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.100Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:49:19.101Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.101Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.101Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.101Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917"}
{"level":"error","ts":"2026-09-18T09:49:19.101Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"cf4e6f3e-c73a-4eee-ab89-52651eee7917","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.141Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308"}
{"level":"info","ts":"2026-09-18T09:49:19.141Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308"}
{"level":"info","ts":"2026-09-18T09:49:19.141Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308"}
{"level":"info","ts":"2026-09-18T09:49:19.141Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:19.141Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308"}
{"level":"info","ts":"2026-09-18T09:49:19.141Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.141Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308"}
{"level":"info","ts":"2026-09-18T09:49:19.141Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:19.141Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308","cluster":"fail-bcffx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.141Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308","cluster":"fail-bcffx","authType":"IntegratedOAuth","serviceAccount":"fail-bcffx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:49:19.141Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-bcffx","ray.io/group":"headgroup","ray.io/identifier":"fail-bcffx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:49:19.141Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:49:19.141Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.141Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.141Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308","enabled":true}
{"level":"error","ts":"2026-09-18T09:49:19.145Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.145Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:49:19.145Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.145Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.145Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.145Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308"}
{"level":"error","ts":"2026-09-18T09:49:19.145Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"0929f85b-34c3-430b-b550-a223518af308","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.156Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57fd08d1-79d9-472d-8011-7e7a7c2f6755"}
{"level":"info","ts":"2026-09-18T09:49:19.156Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57fd08d1-79d9-472d-8011-7e7a7c2f6755"}
{"level":"info","ts":"2026-09-18T09:49:19.156Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57fd08d1-79d9-472d-8011-7e7a7c2f6755"}
{"level":"info","ts":"2026-09-18T09:49:19.156Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57fd08d1-79d9-472d-8011-7e7a7c2f6755","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:19.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57fd08d1-79d9-472d-8011-7e7a7c2f6755","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:19.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57fd08d1-79d9-472d-8011-7e7a7c2f6755","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:19.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57fd08d1-79d9-472d-8011-7e7a7c2f6755","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:19.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57fd08d1-79d9-472d-8011-7e7a7c2f6755","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:19.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57fd08d1-79d9-472d-8011-7e7a7c2f6755","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:19.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57fd08d1-79d9-472d-8011-7e7a7c2f6755","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:19.156Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57fd08d1-79d9-472d-8011-7e7a7c2f6755","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:19.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57fd08d1-79d9-472d-8011-7e7a7c2f6755","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:19.156Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57fd08d1-79d9-472d-8011-7e7a7c2f6755","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:19.156Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57fd08d1-79d9-472d-8011-7e7a7c2f6755","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:19.156Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57fd08d1-79d9-472d-8011-7e7a7c2f6755","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:19.156Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"57fd08d1-79d9-472d-8011-7e7a7c2f6755","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:19.168Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4458ea87-53a3-4b7a-a451-72a166214617","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4458ea87-53a3-4b7a-a451-72a166214617","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4458ea87-53a3-4b7a-a451-72a166214617","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4458ea87-53a3-4b7a-a451-72a166214617"}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4458ea87-53a3-4b7a-a451-72a166214617","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4458ea87-53a3-4b7a-a451-72a166214617","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f9ab1bae-7996-4219-8e30-cb4988b24adb"}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f9ab1bae-7996-4219-8e30-cb4988b24adb"}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f9ab1bae-7996-4219-8e30-cb4988b24adb"}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f9ab1bae-7996-4219-8e30-cb4988b24adb","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f9ab1bae-7996-4219-8e30-cb4988b24adb","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f9ab1bae-7996-4219-8e30-cb4988b24adb","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f9ab1bae-7996-4219-8e30-cb4988b24adb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f9ab1bae-7996-4219-8e30-cb4988b24adb","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f9ab1bae-7996-4219-8e30-cb4988b24adb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f9ab1bae-7996-4219-8e30-cb4988b24adb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f9ab1bae-7996-4219-8e30-cb4988b24adb","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f9ab1bae-7996-4219-8e30-cb4988b24adb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f9ab1bae-7996-4219-8e30-cb4988b24adb","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f9ab1bae-7996-4219-8e30-cb4988b24adb","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f9ab1bae-7996-4219-8e30-cb4988b24adb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:19.169Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f9ab1bae-7996-4219-8e30-cb4988b24adb","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:19.226Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb"}
{"level":"info","ts":"2026-09-18T09:49:19.226Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb"}
{"level":"info","ts":"2026-09-18T09:49:19.226Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb"}
{"level":"info","ts":"2026-09-18T09:49:19.226Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:19.226Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb"}
{"level":"info","ts":"2026-09-18T09:49:19.226Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.226Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb"}
{"level":"info","ts":"2026-09-18T09:49:19.226Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:19.226Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb","cluster":"fail-bcffx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.226Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb","cluster":"fail-bcffx","authType":"IntegratedOAuth","serviceAccount":"fail-bcffx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:49:19.226Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-bcffx","ray.io/group":"headgroup","ray.io/identifier":"fail-bcffx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:49:19.226Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:49:19.226Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.226Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.226Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb","enabled":true}
{"level":"error","ts":"2026-09-18T09:49:19.231Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.231Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:49:19.232Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.232Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.232Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.232Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb"}
{"level":"error","ts":"2026-09-18T09:49:19.232Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"66a9828f-4dbd-493e-979b-dac5a3163cdb","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.296Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"033af31d-7e4b-48eb-b311-575e329842e9"}
{"level":"info","ts":"2026-09-18T09:49:19.296Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"033af31d-7e4b-48eb-b311-575e329842e9"}
{"level":"info","ts":"2026-09-18T09:49:19.296Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"033af31d-7e4b-48eb-b311-575e329842e9"}
{"level":"info","ts":"2026-09-18T09:49:19.296Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"033af31d-7e4b-48eb-b311-575e329842e9","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:19.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"033af31d-7e4b-48eb-b311-575e329842e9","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:19.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"033af31d-7e4b-48eb-b311-575e329842e9","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:19.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"033af31d-7e4b-48eb-b311-575e329842e9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:19.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"033af31d-7e4b-48eb-b311-575e329842e9","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:19.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"033af31d-7e4b-48eb-b311-575e329842e9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:19.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"033af31d-7e4b-48eb-b311-575e329842e9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:19.296Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"033af31d-7e4b-48eb-b311-575e329842e9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:19.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"033af31d-7e4b-48eb-b311-575e329842e9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:19.296Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"033af31d-7e4b-48eb-b311-575e329842e9","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:19.296Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"033af31d-7e4b-48eb-b311-575e329842e9","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:19.297Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"033af31d-7e4b-48eb-b311-575e329842e9","name":"long-running-2qcbf","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0"}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0"}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0"}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0"}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0"}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0","cluster":"fail-bcffx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0","cluster":"fail-bcffx","authType":"IntegratedOAuth","serviceAccount":"fail-bcffx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-bcffx","ray.io/group":"headgroup","ray.io/identifier":"fail-bcffx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0","enabled":true}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"5bd406de-e6e0-4e92-b903-376fc4249983","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"5bd406de-e6e0-4e92-b903-376fc4249983","pod":"long-running-2qcbf-head-cz5d7","ip":"10.132.0.34"}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"9c27031d-3bb0-4968-82b2-7b8b970aee64","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"9c27031d-3bb0-4968-82b2-7b8b970aee64","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:19.324Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"9c27031d-3bb0-4968-82b2-7b8b970aee64","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:19.325Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"9c27031d-3bb0-4968-82b2-7b8b970aee64"}
{"level":"info","ts":"2026-09-18T09:49:19.325Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"9c27031d-3bb0-4968-82b2-7b8b970aee64","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:19.325Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"cb37807f-7381-4256-9e8d-2fd3d8b8a010","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:19.325Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"9c27031d-3bb0-4968-82b2-7b8b970aee64","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:19.325Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"9c27031d-3bb0-4968-82b2-7b8b970aee64","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"error","ts":"2026-09-18T09:49:19.329Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.329Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0"}
{"level":"error","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"09898605-8c74-4627-bab5-b0f3aca39dc0","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df"}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df"}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df"}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df"}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df"}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df","cluster":"fail-bcffx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df","cluster":"fail-bcffx","authType":"IntegratedOAuth","serviceAccount":"fail-bcffx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-bcffx","ray.io/group":"headgroup","ray.io/identifier":"fail-bcffx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.330Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df","enabled":true}
{"level":"error","ts":"2026-09-18T09:49:19.338Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.338Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:49:19.338Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.338Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.338Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.338Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df"}
{"level":"error","ts":"2026-09-18T09:49:19.338Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"23386088-67ed-4a01-b52a-8f77476405df","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.341Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"cb37807f-7381-4256-9e8d-2fd3d8b8a010","name":"long-running-2qcbf-head"}
{"level":"info","ts":"2026-09-18T09:49:19.342Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"5bd406de-e6e0-4e92-b903-376fc4249983","certificate":"ray-head-cert-long-running-2qcbf","dnsNames":["long-running-2qcbf-head-svc","localhost","long-running-2qcbf-head-svc.test-ns-chvpt.svc","long-running-2qcbf-head-svc.test-ns-chvpt.svc.cluster.local"],"ipAddresses":["10.132.0.34","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:49:19.347Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"accdc716-a53c-4665-9166-fee4df812a21","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:19.347Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"accdc716-a53c-4665-9166-fee4df812a21","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:19.347Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"accdc716-a53c-4665-9166-fee4df812a21","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:19.347Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"accdc716-a53c-4665-9166-fee4df812a21"}
{"level":"info","ts":"2026-09-18T09:49:19.347Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"accdc716-a53c-4665-9166-fee4df812a21","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:19.347Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"accdc716-a53c-4665-9166-fee4df812a21","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:19.356Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"cb37807f-7381-4256-9e8d-2fd3d8b8a010","name":"long-running-2qcbf-workers"}
{"level":"info","ts":"2026-09-18T09:49:19.356Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"cb37807f-7381-4256-9e8d-2fd3d8b8a010","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:19.360Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"5bd406de-e6e0-4e92-b903-376fc4249983","certificate":"ray-worker-cert-long-running-2qcbf","dnsNames":["long-running-2qcbf-worker-svc","localhost","long-running-2qcbf-small-group","long-running-2qcbf-worker-svc.test-ns-chvpt.svc","long-running-2qcbf-worker-svc.test-ns-chvpt.svc.cluster.local","*.long-running-2qcbf-worker-svc.test-ns-chvpt.svc","*.long-running-2qcbf-worker-svc.test-ns-chvpt.svc.cluster.local","*-worker-*.test-ns-chvpt.svc","*-worker-*.test-ns-chvpt.svc.cluster.local"],"ipAddresses":["10.132.0.34","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:49:19.360Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"5bd406de-e6e0-4e92-b903-376fc4249983","rayCluster":"long-running-2qcbf","podIPs":["10.132.0.34"]}
{"level":"info","ts":"2026-09-18T09:49:19.392Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3"}
{"level":"info","ts":"2026-09-18T09:49:19.392Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3"}
{"level":"info","ts":"2026-09-18T09:49:19.392Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3"}
{"level":"info","ts":"2026-09-18T09:49:19.392Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:19.393Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3"}
{"level":"info","ts":"2026-09-18T09:49:19.393Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.393Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3"}
{"level":"info","ts":"2026-09-18T09:49:19.393Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:19.393Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3","cluster":"fail-bcffx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:19.393Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3","cluster":"fail-bcffx","authType":"IntegratedOAuth","serviceAccount":"fail-bcffx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:49:19.393Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-bcffx","ray.io/group":"headgroup","ray.io/identifier":"fail-bcffx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:49:19.393Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:49:19.393Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.393Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:19.393Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3","enabled":true}
{"level":"error","ts":"2026-09-18T09:49:19.398Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:19.398Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:49:19.398Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.398Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.398Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:19.398Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3"}
{"level":"error","ts":"2026-09-18T09:49:19.398Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"3cca14a5-d746-467b-81e6-ba206f90fbc3","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:20.027Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"332cf139-ccf8-4862-8687-3811bcc690bc"}
{"level":"info","ts":"2026-09-18T09:49:20.027Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"332cf139-ccf8-4862-8687-3811bcc690bc"}
{"level":"info","ts":"2026-09-18T09:49:20.027Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"332cf139-ccf8-4862-8687-3811bcc690bc"}
{"level":"info","ts":"2026-09-18T09:49:20.027Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"332cf139-ccf8-4862-8687-3811bcc690bc","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:20.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"332cf139-ccf8-4862-8687-3811bcc690bc","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:20.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"332cf139-ccf8-4862-8687-3811bcc690bc","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:20.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"332cf139-ccf8-4862-8687-3811bcc690bc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:20.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"332cf139-ccf8-4862-8687-3811bcc690bc","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:20.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"332cf139-ccf8-4862-8687-3811bcc690bc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:20.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"332cf139-ccf8-4862-8687-3811bcc690bc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:20.027Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"332cf139-ccf8-4862-8687-3811bcc690bc","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:20.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"332cf139-ccf8-4862-8687-3811bcc690bc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:20.027Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"332cf139-ccf8-4862-8687-3811bcc690bc","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:20.027Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"332cf139-ccf8-4862-8687-3811bcc690bc","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:20.027Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"332cf139-ccf8-4862-8687-3811bcc690bc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:20.027Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"332cf139-ccf8-4862-8687-3811bcc690bc","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:20.582Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05ce1828-2ef8-42e3-923a-230d2d9e8839"}
{"level":"info","ts":"2026-09-18T09:49:20.582Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05ce1828-2ef8-42e3-923a-230d2d9e8839"}
{"level":"info","ts":"2026-09-18T09:49:20.582Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05ce1828-2ef8-42e3-923a-230d2d9e8839"}
{"level":"info","ts":"2026-09-18T09:49:20.582Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05ce1828-2ef8-42e3-923a-230d2d9e8839","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:20.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05ce1828-2ef8-42e3-923a-230d2d9e8839","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:20.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05ce1828-2ef8-42e3-923a-230d2d9e8839","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:20.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05ce1828-2ef8-42e3-923a-230d2d9e8839","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:20.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05ce1828-2ef8-42e3-923a-230d2d9e8839","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:20.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05ce1828-2ef8-42e3-923a-230d2d9e8839","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:20.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05ce1828-2ef8-42e3-923a-230d2d9e8839","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:20.582Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05ce1828-2ef8-42e3-923a-230d2d9e8839","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:20.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05ce1828-2ef8-42e3-923a-230d2d9e8839","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:20.582Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05ce1828-2ef8-42e3-923a-230d2d9e8839","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:20.582Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05ce1828-2ef8-42e3-923a-230d2d9e8839","podName":"long-running-2qcbf-head-cz5d7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:20.582Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05ce1828-2ef8-42e3-923a-230d2d9e8839","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:20.582Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"05ce1828-2ef8-42e3-923a-230d2d9e8839","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:20.679Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61"}
{"level":"info","ts":"2026-09-18T09:49:20.679Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61"}
{"level":"info","ts":"2026-09-18T09:49:20.679Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61"}
{"level":"info","ts":"2026-09-18T09:49:20.679Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:20.679Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61"}
{"level":"info","ts":"2026-09-18T09:49:20.679Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:20.679Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61"}
{"level":"info","ts":"2026-09-18T09:49:20.679Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:20.679Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61","cluster":"fail-bcffx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:20.679Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61","cluster":"fail-bcffx","authType":"IntegratedOAuth","serviceAccount":"fail-bcffx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:49:20.679Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-bcffx","ray.io/group":"headgroup","ray.io/identifier":"fail-bcffx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:49:20.679Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:49:20.679Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:20.679Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:20.679Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61","enabled":true}
{"level":"error","ts":"2026-09-18T09:49:20.683Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:20.683Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:49:20.683Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:20.683Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:20.684Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:20.684Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61"}
{"level":"error","ts":"2026-09-18T09:49:20.684Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"e7611e95-39dd-4d12-b345-3dc4eac52e61","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:22.170Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"90c1c9da-2e44-44d0-b728-28dae2ea8379","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:22.170Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"90c1c9da-2e44-44d0-b728-28dae2ea8379","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:22.170Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"90c1c9da-2e44-44d0-b728-28dae2ea8379","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:22.170Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"90c1c9da-2e44-44d0-b728-28dae2ea8379"}
{"level":"info","ts":"2026-09-18T09:49:22.170Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"90c1c9da-2e44-44d0-b728-28dae2ea8379","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:22.170Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"90c1c9da-2e44-44d0-b728-28dae2ea8379","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:23.244Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b"}
{"level":"info","ts":"2026-09-18T09:49:23.244Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b"}
{"level":"info","ts":"2026-09-18T09:49:23.244Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b"}
{"level":"info","ts":"2026-09-18T09:49:23.244Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b","1 head service found":"fail-bcffx-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:23.244Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b"}
{"level":"info","ts":"2026-09-18T09:49:23.244Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:23.244Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b"}
{"level":"info","ts":"2026-09-18T09:49:23.244Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:23.244Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b","cluster":"fail-bcffx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:49:23.244Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b","cluster":"fail-bcffx","authType":"IntegratedOAuth","serviceAccount":"fail-bcffx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:49:23.244Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-bcffx","ray.io/group":"headgroup","ray.io/identifier":"fail-bcffx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:49:23.244Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:49:23.244Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:23.244Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:49:23.244Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b","enabled":true}
{"level":"error","ts":"2026-09-18T09:49:23.248Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:23.248Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:49:23.248Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:23.248Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:23.248Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:23.248Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b"}
{"level":"error","ts":"2026-09-18T09:49:23.248Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d57b97be-ce80-43e8-a670-026e40f8d82b","error":"FailedCreateHeadPod\npods \"fail-bcffx-head-\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:24.311Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"1817f120-f366-4f6f-8c9b-e3477faec04b"}
{"level":"info","ts":"2026-09-18T09:49:24.311Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"1817f120-f366-4f6f-8c9b-e3477faec04b"}
{"level":"info","ts":"2026-09-18T09:49:24.311Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"1817f120-f366-4f6f-8c9b-e3477faec04b"}
{"level":"info","ts":"2026-09-18T09:49:24.311Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"1817f120-f366-4f6f-8c9b-e3477faec04b","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:24.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"1817f120-f366-4f6f-8c9b-e3477faec04b","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:24.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"1817f120-f366-4f6f-8c9b-e3477faec04b","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:24.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"1817f120-f366-4f6f-8c9b-e3477faec04b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:24.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"1817f120-f366-4f6f-8c9b-e3477faec04b","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:24.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"1817f120-f366-4f6f-8c9b-e3477faec04b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:24.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"1817f120-f366-4f6f-8c9b-e3477faec04b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:24.312Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"1817f120-f366-4f6f-8c9b-e3477faec04b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:24.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"1817f120-f366-4f6f-8c9b-e3477faec04b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:24.312Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"1817f120-f366-4f6f-8c9b-e3477faec04b","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:24.312Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"1817f120-f366-4f6f-8c9b-e3477faec04b","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:24.312Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"1817f120-f366-4f6f-8c9b-e3477faec04b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:24.312Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"1817f120-f366-4f6f-8c9b-e3477faec04b","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:25.036Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"9bfa1749-94dd-4de1-9a48-a70089fb0d4e","namespacedName":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:25.036Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"9bfa1749-94dd-4de1-9a48-a70089fb0d4e","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.036Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"9bfa1749-94dd-4de1-9a48-a70089fb0d4e","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.062Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"203d29ac-a0cf-4021-8cea-4434f2bab839","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.062Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"6f5fb04e-89a2-4053-9549-3bbcb55b118f","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:25.062Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"443eb3cd-3c0d-409f-b866-33b79b5acdf4","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.062Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"443eb3cd-3c0d-409f-b866-33b79b5acdf4","rayCluster":"fail-bcffx"}
{"level":"error","ts":"2026-09-18T09:49:25.063Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"9bfa1749-94dd-4de1-9a48-a70089fb0d4e","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"fail-bcffx-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.063Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"9bfa1749-94dd-4de1-9a48-a70089fb0d4e"}
{"level":"error","ts":"2026-09-18T09:49:25.063Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"9bfa1749-94dd-4de1-9a48-a70089fb0d4e","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"fail-bcffx-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:49:25.067Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"443eb3cd-3c0d-409f-b866-33b79b5acdf4","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:49:25.067Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"443eb3cd-3c0d-409f-b866-33b79b5acdf4","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.067Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"443eb3cd-3c0d-409f-b866-33b79b5acdf4"}
{"level":"error","ts":"2026-09-18T09:49:25.067Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"443eb3cd-3c0d-409f-b866-33b79b5acdf4","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.069Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"dd818fa9-41db-4d7d-afa7-a96e5f9e3b42","namespacedName":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:25.069Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"dd818fa9-41db-4d7d-afa7-a96e5f9e3b42","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.069Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"dd818fa9-41db-4d7d-afa7-a96e5f9e3b42","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.069Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"203d29ac-a0cf-4021-8cea-4434f2bab839","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.072Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"6c5e476b-8a58-4e5d-aaa0-4ca114db29a7","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.072Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"6c5e476b-8a58-4e5d-aaa0-4ca114db29a7","rayCluster":"fail-bcffx"}
{"level":"error","ts":"2026-09-18T09:49:25.075Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"6c5e476b-8a58-4e5d-aaa0-4ca114db29a7","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:49:25.075Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"6c5e476b-8a58-4e5d-aaa0-4ca114db29a7","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.075Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"6c5e476b-8a58-4e5d-aaa0-4ca114db29a7"}
{"level":"error","ts":"2026-09-18T09:49:25.075Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"6c5e476b-8a58-4e5d-aaa0-4ca114db29a7","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.085Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"dc97c086-eba1-4926-b46d-b5b626d7324d","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.085Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"dc97c086-eba1-4926-b46d-b5b626d7324d","rayCluster":"fail-bcffx"}
{"level":"error","ts":"2026-09-18T09:49:25.087Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"dd818fa9-41db-4d7d-afa7-a96e5f9e3b42","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"fail-bcffx-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.087Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"dd818fa9-41db-4d7d-afa7-a96e5f9e3b42"}
{"level":"error","ts":"2026-09-18T09:49:25.087Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"dd818fa9-41db-4d7d-afa7-a96e5f9e3b42","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"fail-bcffx-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:49:25.090Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"dc97c086-eba1-4926-b46d-b5b626d7324d","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:49:25.091Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"dc97c086-eba1-4926-b46d-b5b626d7324d","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.091Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"dc97c086-eba1-4926-b46d-b5b626d7324d"}
{"level":"error","ts":"2026-09-18T09:49:25.091Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"dc97c086-eba1-4926-b46d-b5b626d7324d","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.098Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"7ce8a35b-99a0-4cdd-bef8-d2d884cb9149","namespacedName":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:25.098Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"7ce8a35b-99a0-4cdd-bef8-d2d884cb9149","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.098Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"7ce8a35b-99a0-4cdd-bef8-d2d884cb9149","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.112Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"7b1ad25b-b5bb-4655-8c2e-79e72972e830","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.112Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"7b1ad25b-b5bb-4655-8c2e-79e72972e830","rayCluster":"fail-bcffx"}
{"level":"error","ts":"2026-09-18T09:49:25.115Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"7b1ad25b-b5bb-4655-8c2e-79e72972e830","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:49:25.115Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"7b1ad25b-b5bb-4655-8c2e-79e72972e830","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.115Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"7b1ad25b-b5bb-4655-8c2e-79e72972e830"}
{"level":"error","ts":"2026-09-18T09:49:25.115Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"7b1ad25b-b5bb-4655-8c2e-79e72972e830","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:49:25.123Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"7ce8a35b-99a0-4cdd-bef8-d2d884cb9149","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"fail-bcffx-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.123Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"7ce8a35b-99a0-4cdd-bef8-d2d884cb9149"}
{"level":"error","ts":"2026-09-18T09:49:25.123Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"7ce8a35b-99a0-4cdd-bef8-d2d884cb9149","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"fail-bcffx-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.144Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"6b855f36-eef3-4082-b7f6-47890ebbf6bf","namespacedName":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:25.144Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"6b855f36-eef3-4082-b7f6-47890ebbf6bf","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.144Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"6b855f36-eef3-4082-b7f6-47890ebbf6bf","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.155Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"60907160-20a9-42cf-9138-5611ef84b4f5","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.155Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"60907160-20a9-42cf-9138-5611ef84b4f5","rayCluster":"fail-bcffx"}
{"level":"error","ts":"2026-09-18T09:49:25.158Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"60907160-20a9-42cf-9138-5611ef84b4f5","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:49:25.158Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"60907160-20a9-42cf-9138-5611ef84b4f5","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.158Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"60907160-20a9-42cf-9138-5611ef84b4f5"}
{"level":"error","ts":"2026-09-18T09:49:25.158Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"60907160-20a9-42cf-9138-5611ef84b4f5","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:49:25.160Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"6b855f36-eef3-4082-b7f6-47890ebbf6bf","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"fail-bcffx-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.160Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"6b855f36-eef3-4082-b7f6-47890ebbf6bf"}
{"level":"error","ts":"2026-09-18T09:49:25.160Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"6b855f36-eef3-4082-b7f6-47890ebbf6bf","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"fail-bcffx-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.171Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb1a2b9c-e968-453b-9730-1421aae31fb5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:25.171Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb1a2b9c-e968-453b-9730-1421aae31fb5","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:25.171Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb1a2b9c-e968-453b-9730-1421aae31fb5","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:25.171Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb1a2b9c-e968-453b-9730-1421aae31fb5"}
{"level":"info","ts":"2026-09-18T09:49:25.171Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb1a2b9c-e968-453b-9730-1421aae31fb5","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:25.171Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb1a2b9c-e968-453b-9730-1421aae31fb5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:25.200Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"a759a091-0d61-45a7-b7fa-589843745050","namespacedName":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:25.200Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"a759a091-0d61-45a7-b7fa-589843745050","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.200Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"a759a091-0d61-45a7-b7fa-589843745050","cluster":"fail-bcffx"}
{"level":"error","ts":"2026-09-18T09:49:25.214Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"a759a091-0d61-45a7-b7fa-589843745050","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"fail-bcffx-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.214Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"a759a091-0d61-45a7-b7fa-589843745050"}
{"level":"error","ts":"2026-09-18T09:49:25.214Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"a759a091-0d61-45a7-b7fa-589843745050","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"fail-bcffx-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.239Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"4f8cdce0-2e00-4f05-8e88-2136d0365352","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.239Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"4f8cdce0-2e00-4f05-8e88-2136d0365352","rayCluster":"fail-bcffx"}
{"level":"error","ts":"2026-09-18T09:49:25.242Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"4f8cdce0-2e00-4f05-8e88-2136d0365352","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:49:25.242Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"4f8cdce0-2e00-4f05-8e88-2136d0365352","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.242Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"4f8cdce0-2e00-4f05-8e88-2136d0365352"}
{"level":"error","ts":"2026-09-18T09:49:25.242Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"4f8cdce0-2e00-4f05-8e88-2136d0365352","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.278Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"7333e2fc-cc03-4e34-8f9b-b40a85a589e4","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:25.278Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"7333e2fc-cc03-4e34-8f9b-b40a85a589e4","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:49:12 +0000 UTC","Now":"2026-09-18T09:49:25.278Z","ShutdownTime":"2026-09-18T09:49:12.000Z"}
{"level":"info","ts":"2026-09-18T09:49:25.278Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"7333e2fc-cc03-4e34-8f9b-b40a85a589e4","RayCluster":{"name":"counter-auth-6hjnc","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:25.278Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"7333e2fc-cc03-4e34-8f9b-b40a85a589e4","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T09:49:25.278Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"7333e2fc-cc03-4e34-8f9b-b40a85a589e4","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-6hjnc"}
{"level":"info","ts":"2026-09-18T09:49:25.283Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b230a739-c96f-4bd3-946c-2692df4aa777","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-09-18T09:49:25.283Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b230a739-c96f-4bd3-946c-2692df4aa777","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:45:59 +0000 UTC","Now":"2026-09-18T09:49:25.283Z","ShutdownTime":"2026-09-18T09:45:59.000Z"}
{"level":"info","ts":"2026-09-18T09:49:25.283Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b230a739-c96f-4bd3-946c-2692df4aa777","RayCluster":{"name":"counter-wmhkk","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:25.283Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b230a739-c96f-4bd3-946c-2692df4aa777","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T09:49:25.283Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"b230a739-c96f-4bd3-946c-2692df4aa777","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-wmhkk"}
{"level":"info","ts":"2026-09-18T09:49:25.287Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"273c1425-90bd-4112-9fc5-ca3c8d0e3ebd","namespacedName":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:25.287Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"273c1425-90bd-4112-9fc5-ca3c8d0e3ebd","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.287Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"273c1425-90bd-4112-9fc5-ca3c8d0e3ebd","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.287Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d8d2be99-fcdb-4eef-8a39-3ec8d5dff8ad"}
{"level":"info","ts":"2026-09-18T09:49:25.287Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d8d2be99-fcdb-4eef-8a39-3ec8d5dff8ad"}
{"level":"info","ts":"2026-09-18T09:49:25.287Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d8d2be99-fcdb-4eef-8a39-3ec8d5dff8ad"}
{"level":"error","ts":"2026-09-18T09:49:25.290Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d8d2be99-fcdb-4eef-8a39-3ec8d5dff8ad","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcileHeadService-fm","error":"services \"fail-bcffx-head-svc\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.290Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d8d2be99-fcdb-4eef-8a39-3ec8d5dff8ad","error":"services \"fail-bcffx-head-svc\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated"}
{"level":"info","ts":"2026-09-18T09:49:25.290Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d8d2be99-fcdb-4eef-8a39-3ec8d5dff8ad","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:25.290Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d8d2be99-fcdb-4eef-8a39-3ec8d5dff8ad","serviceSelectors":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/identifier":"fail-bcffx-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:25.290Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d8d2be99-fcdb-4eef-8a39-3ec8d5dff8ad","filter labels":["test-ns-q2lj2",{"ray.io/cluster":"fail-bcffx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:49:25.290Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d8d2be99-fcdb-4eef-8a39-3ec8d5dff8ad","error":"unable to find head service. cluster name fail-bcffx, filter labels [test-ns-q2lj2 map[ray.io/cluster:fail-bcffx ray.io/identifier:fail-bcffx-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-09-18T09:49:25.290Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d8d2be99-fcdb-4eef-8a39-3ec8d5dff8ad"}
{"level":"error","ts":"2026-09-18T09:49:25.290Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"d8d2be99-fcdb-4eef-8a39-3ec8d5dff8ad","error":"services \"fail-bcffx-head-svc\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.290Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"f0f7394d-3dfd-4e4a-b712-fd32cc5b5960","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"error","ts":"2026-09-18T09:49:25.305Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"273c1425-90bd-4112-9fc5-ca3c8d0e3ebd","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"fail-bcffx-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.305Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"273c1425-90bd-4112-9fc5-ca3c8d0e3ebd"}
{"level":"error","ts":"2026-09-18T09:49:25.305Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"273c1425-90bd-4112-9fc5-ca3c8d0e3ebd","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"fail-bcffx-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.305Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"38bd9251-dfed-4468-949c-e42ad416ee4d","namespacedName":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:25.305Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"38bd9251-dfed-4468-949c-e42ad416ee4d","mode":"IntegratedOAuth","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.305Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"38bd9251-dfed-4468-949c-e42ad416ee4d","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.321Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f21d4d39-e0ad-4b73-b745-849078b184e9"}
{"level":"info","ts":"2026-09-18T09:49:25.321Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f21d4d39-e0ad-4b73-b745-849078b184e9"}
{"level":"info","ts":"2026-09-18T09:49:25.321Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f21d4d39-e0ad-4b73-b745-849078b184e9"}
{"level":"info","ts":"2026-09-18T09:49:25.321Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f21d4d39-e0ad-4b73-b745-849078b184e9","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:25.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f21d4d39-e0ad-4b73-b745-849078b184e9","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:25.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f21d4d39-e0ad-4b73-b745-849078b184e9","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:25.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f21d4d39-e0ad-4b73-b745-849078b184e9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:25.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f21d4d39-e0ad-4b73-b745-849078b184e9","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:25.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f21d4d39-e0ad-4b73-b745-849078b184e9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:25.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f21d4d39-e0ad-4b73-b745-849078b184e9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:25.321Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f21d4d39-e0ad-4b73-b745-849078b184e9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:25.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f21d4d39-e0ad-4b73-b745-849078b184e9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:25.322Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f21d4d39-e0ad-4b73-b745-849078b184e9","podName":"long-running-2qcbf-head-cz5d7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:49:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:49:25.322Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f21d4d39-e0ad-4b73-b745-849078b184e9","podName":"long-running-2qcbf-head-cz5d7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:49:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:49:25.322Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f21d4d39-e0ad-4b73-b745-849078b184e9","name":"long-running-2qcbf","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"error","ts":"2026-09-18T09:49:25.323Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"38bd9251-dfed-4468-949c-e42ad416ee4d","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"fail-bcffx-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.323Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"38bd9251-dfed-4468-949c-e42ad416ee4d"}
{"level":"error","ts":"2026-09-18T09:49:25.323Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"38bd9251-dfed-4468-949c-e42ad416ee4d","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"fail-bcffx-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.338Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"94778cec-5f8a-4455-8c92-0354f2d4e4ed","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:25.338Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"9b10d68e-1c0b-4aef-9f43-42f47bfa24de","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:25.338Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"9b10d68e-1c0b-4aef-9f43-42f47bfa24de","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:25.338Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"9b10d68e-1c0b-4aef-9f43-42f47bfa24de","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:25.338Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"d5b9dce2-8832-4cb2-81b9-6faa72a55f23","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:25.338Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"9b10d68e-1c0b-4aef-9f43-42f47bfa24de"}
{"level":"info","ts":"2026-09-18T09:49:25.338Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"9b10d68e-1c0b-4aef-9f43-42f47bfa24de","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:25.338Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"9b10d68e-1c0b-4aef-9f43-42f47bfa24de","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:25.338Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"9b10d68e-1c0b-4aef-9f43-42f47bfa24de","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:49:25.338Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"d5b9dce2-8832-4cb2-81b9-6faa72a55f23","pod":"long-running-2qcbf-head-cz5d7","ip":"10.132.0.34"}
{"level":"info","ts":"2026-09-18T09:49:25.338Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"d5b9dce2-8832-4cb2-81b9-6faa72a55f23","pod":"long-running-2qcbf-small-group-worker-94s9h","ip":"10.133.0.32"}
{"level":"info","ts":"2026-09-18T09:49:25.353Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"d5b9dce2-8832-4cb2-81b9-6faa72a55f23","certificate":"ray-head-cert-long-running-2qcbf","dnsNames":["long-running-2qcbf-head-svc","localhost","long-running-2qcbf-head-svc.test-ns-chvpt.svc","long-running-2qcbf-head-svc.test-ns-chvpt.svc.cluster.local"],"ipAddresses":["10.132.0.34","10.133.0.32","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:49:25.353Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"44cbc61b-e670-4a02-98cc-f163a816660f","DeletionTimestamp":"2026-09-18 09:49:25 +0000 UTC"}
{"level":"info","ts":"2026-09-18T09:49:25.353Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"44cbc61b-e670-4a02-98cc-f163a816660f","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:49:25.366Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"94778cec-5f8a-4455-8c92-0354f2d4e4ed","name":"long-running-2qcbf-head"}
{"level":"info","ts":"2026-09-18T09:49:25.367Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"d5b9dce2-8832-4cb2-81b9-6faa72a55f23","certificate":"ray-worker-cert-long-running-2qcbf","dnsNames":["long-running-2qcbf-worker-svc","localhost","long-running-2qcbf-small-group","long-running-2qcbf-worker-svc.test-ns-chvpt.svc","long-running-2qcbf-worker-svc.test-ns-chvpt.svc.cluster.local","*.long-running-2qcbf-worker-svc.test-ns-chvpt.svc","*.long-running-2qcbf-worker-svc.test-ns-chvpt.svc.cluster.local","*-worker-*.test-ns-chvpt.svc","*-worker-*.test-ns-chvpt.svc.cluster.local"],"ipAddresses":["10.132.0.34","10.133.0.32","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:49:25.367Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"d5b9dce2-8832-4cb2-81b9-6faa72a55f23","rayCluster":"long-running-2qcbf","podIPs":["10.132.0.34","10.133.0.32"]}
{"level":"info","ts":"2026-09-18T09:49:25.373Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b62b0329-aadf-4b03-9855-f723217a424e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:25.373Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b62b0329-aadf-4b03-9855-f723217a424e","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:25.373Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b62b0329-aadf-4b03-9855-f723217a424e","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:25.373Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b62b0329-aadf-4b03-9855-f723217a424e"}
{"level":"info","ts":"2026-09-18T09:49:25.373Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b62b0329-aadf-4b03-9855-f723217a424e","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:25.373Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b62b0329-aadf-4b03-9855-f723217a424e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:25.373Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"ac288745-403e-4d54-b4dc-ee65d577c159","DeletionTimestamp":"2026-09-18 09:49:25 +0000 UTC"}
{"level":"info","ts":"2026-09-18T09:49:25.373Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"ac288745-403e-4d54-b4dc-ee65d577c159","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:49:25.380Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"94778cec-5f8a-4455-8c92-0354f2d4e4ed","name":"long-running-2qcbf-workers"}
{"level":"info","ts":"2026-09-18T09:49:25.380Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"94778cec-5f8a-4455-8c92-0354f2d4e4ed","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:25.396Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"0ce3deea-dc9a-4b3e-8c3d-226905158e0d","DeletionTimestamp":"2026-09-18 09:49:25 +0000 UTC"}
{"level":"info","ts":"2026-09-18T09:49:25.396Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"0ce3deea-dc9a-4b3e-8c3d-226905158e0d","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:49:25.403Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"e734c172-06c5-4e7f-8b47-49308390121e","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.403Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"e734c172-06c5-4e7f-8b47-49308390121e","rayCluster":"fail-bcffx"}
{"level":"error","ts":"2026-09-18T09:49:25.406Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"e734c172-06c5-4e7f-8b47-49308390121e","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:49:25.406Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"e734c172-06c5-4e7f-8b47-49308390121e","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.406Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"e734c172-06c5-4e7f-8b47-49308390121e"}
{"level":"error","ts":"2026-09-18T09:49:25.406Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"e734c172-06c5-4e7f-8b47-49308390121e","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.411Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"47b0fc99-bca5-409a-a74e-55cc52dc3b99"}
{"level":"info","ts":"2026-09-18T09:49:25.411Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"6d31a16b-6c73-4a5a-991b-a21006dfd90a"}
{"level":"info","ts":"2026-09-18T09:49:25.411Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"bd80a2b0-4d83-43ec-a9e8-0080fd3fe90d"}
{"level":"info","ts":"2026-09-18T09:49:25.422Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"d5063523-8919-4865-a90a-209d066b8344"}
{"level":"info","ts":"2026-09-18T09:49:25.422Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"reconcileID":"70e2edaf-6cbf-484e-b382-880c7c84b879"}
{"level":"info","ts":"2026-09-18T09:49:25.422Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"95e3cb68-b895-4206-9828-385cba1e617b","namespacedName":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:25.422Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"95e3cb68-b895-4206-9828-385cba1e617b","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.422Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"95e3cb68-b895-4206-9828-385cba1e617b","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.422Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"332dcfe9-7acf-4c93-8a4a-8d2fd052e637","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.422Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"332dcfe9-7acf-4c93-8a4a-8d2fd052e637","rayCluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.422Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"95e3cb68-b895-4206-9828-385cba1e617b","namespace":"test-ns-q2lj2","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T09:49:25.422Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"b20e2003-1049-46a7-ab62-d86fb9c4aed8"}
{"level":"error","ts":"2026-09-18T09:49:25.425Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"332dcfe9-7acf-4c93-8a4a-8d2fd052e637","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:49:25.426Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"332dcfe9-7acf-4c93-8a4a-8d2fd052e637","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.426Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"332dcfe9-7acf-4c93-8a4a-8d2fd052e637"}
{"level":"error","ts":"2026-09-18T09:49:25.426Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"332dcfe9-7acf-4c93-8a4a-8d2fd052e637","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-bcffx\" is forbidden: unable to create new content in namespace test-ns-q2lj2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.466Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"95e3cb68-b895-4206-9828-385cba1e617b","name":"kuberay-gateway-access","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:49:25.475Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"95e3cb68-b895-4206-9828-385cba1e617b","name":"test-ns-q2lj2-fail-bcffx","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T09:49:25.483Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"95e3cb68-b895-4206-9828-385cba1e617b","configMap":"kube-rbac-proxy-config-fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.491Z","logger":"raycluster-resource","msg":"default","name":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.491Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-bcffx","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:49:25.509Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"95e3cb68-b895-4206-9828-385cba1e617b","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.509Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"d3351b61-5bd9-4805-af58-47bf7285de5d","namespacedName":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:25.509Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"d3351b61-5bd9-4805-af58-47bf7285de5d","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.509Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"d3351b61-5bd9-4805-af58-47bf7285de5d","cluster":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.509Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"d3351b61-5bd9-4805-af58-47bf7285de5d","namespace":"test-ns-q2lj2","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T09:49:25.509Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"d3351b61-5bd9-4805-af58-47bf7285de5d","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-09-18T09:49:25.510Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"13ed75ed-fb4a-47c8-89fd-a332b888f5ba"}
{"level":"info","ts":"2026-09-18T09:49:25.510Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"552e48bc-7fb0-49f3-a7b5-825c8a5aebcd","namespace":"test-ns-q2lj2","clusterName":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.510Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"379f08ef-870d-4840-9af4-eaccf456d722"}
{"level":"error","ts":"2026-09-18T09:49:25.513Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"d3351b61-5bd9-4805-af58-47bf7285de5d","error":"rayclusters.ray.io \"fail-bcffx\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:196\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:137\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.513Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"d3351b61-5bd9-4805-af58-47bf7285de5d"}
{"level":"error","ts":"2026-09-18T09:49:25.513Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"d3351b61-5bd9-4805-af58-47bf7285de5d","error":"rayclusters.ray.io \"fail-bcffx\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.514Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"31c0b803-a347-411e-bf39-66ad9d394088","namespacedName":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:25.514Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"31c0b803-a347-411e-bf39-66ad9d394088"}
{"level":"info","ts":"2026-09-18T09:49:25.514Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"31c0b803-a347-411e-bf39-66ad9d394088","namespace":"test-ns-q2lj2","authClusters":0}
{"level":"info","ts":"2026-09-18T09:49:25.514Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"31c0b803-a347-411e-bf39-66ad9d394088","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:49:25.517Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"3b813c99-a1bc-4181-b52e-e1dbfd1f9738"}
{"level":"info","ts":"2026-09-18T09:49:25.519Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c36fdf76-75e4-4f5c-8d69-54b48f6797cb","namespacedName":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"}}
{"level":"info","ts":"2026-09-18T09:49:25.519Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c36fdf76-75e4-4f5c-8d69-54b48f6797cb"}
{"level":"info","ts":"2026-09-18T09:49:25.519Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c36fdf76-75e4-4f5c-8d69-54b48f6797cb","namespace":"test-ns-q2lj2","authClusters":0}
{"level":"info","ts":"2026-09-18T09:49:25.519Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"c36fdf76-75e4-4f5c-8d69-54b48f6797cb","namespace":"test-ns-q2lj2"}
{"level":"info","ts":"2026-09-18T09:49:25.524Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"77c9a285-db88-4b32-bf8e-a97f6a3a8598"}
{"level":"error","ts":"2026-09-18T09:49:25.566Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"552e48bc-7fb0-49f3-a7b5-825c8a5aebcd","issuer":"ray-selfsigned-issuer-fail-bcffx","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-bcffx\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.566Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"552e48bc-7fb0-49f3-a7b5-825c8a5aebcd","errorCount":1}
{"level":"info","ts":"2026-09-18T09:49:25.566Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"552e48bc-7fb0-49f3-a7b5-825c8a5aebcd"}
{"level":"error","ts":"2026-09-18T09:49:25.566Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"552e48bc-7fb0-49f3-a7b5-825c8a5aebcd","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-bcffx\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:49:25.727Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"2aa44bc4-172a-42c9-82e0-2e6cb1d834eb","namespace":"test-ns-q2lj2","clusterName":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:25.727Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-bcffx","namespace":"test-ns-q2lj2"},"namespace":"test-ns-q2lj2","name":"fail-bcffx","reconcileID":"2aa44bc4-172a-42c9-82e0-2e6cb1d834eb","namespace":"test-ns-q2lj2","clusterName":"fail-bcffx"}
{"level":"info","ts":"2026-09-18T09:49:27.337Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5a0a2d9d-df01-4d90-bdc1-7d2c521e33f8"}
{"level":"info","ts":"2026-09-18T09:49:27.337Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5a0a2d9d-df01-4d90-bdc1-7d2c521e33f8"}
{"level":"info","ts":"2026-09-18T09:49:27.337Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5a0a2d9d-df01-4d90-bdc1-7d2c521e33f8"}
{"level":"info","ts":"2026-09-18T09:49:27.337Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5a0a2d9d-df01-4d90-bdc1-7d2c521e33f8","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:27.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5a0a2d9d-df01-4d90-bdc1-7d2c521e33f8","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:27.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5a0a2d9d-df01-4d90-bdc1-7d2c521e33f8","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:27.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5a0a2d9d-df01-4d90-bdc1-7d2c521e33f8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:27.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5a0a2d9d-df01-4d90-bdc1-7d2c521e33f8","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:27.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5a0a2d9d-df01-4d90-bdc1-7d2c521e33f8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:27.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5a0a2d9d-df01-4d90-bdc1-7d2c521e33f8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:27.337Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5a0a2d9d-df01-4d90-bdc1-7d2c521e33f8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:27.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5a0a2d9d-df01-4d90-bdc1-7d2c521e33f8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:27.337Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5a0a2d9d-df01-4d90-bdc1-7d2c521e33f8","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:27.338Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5a0a2d9d-df01-4d90-bdc1-7d2c521e33f8","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:27.338Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5a0a2d9d-df01-4d90-bdc1-7d2c521e33f8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:27.338Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5a0a2d9d-df01-4d90-bdc1-7d2c521e33f8","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:28.171Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72285a45-dc9a-464a-9d23-36e4d16834b0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:28.171Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72285a45-dc9a-464a-9d23-36e4d16834b0","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:28.171Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72285a45-dc9a-464a-9d23-36e4d16834b0","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:28.172Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72285a45-dc9a-464a-9d23-36e4d16834b0"}
{"level":"info","ts":"2026-09-18T09:49:28.172Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72285a45-dc9a-464a-9d23-36e4d16834b0","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:28.172Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72285a45-dc9a-464a-9d23-36e4d16834b0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:28.373Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter","namespace":"test-ns-q2lj2"},"reconcileID":"f468b01a-27d7-4d11-bd85-e1bf27393c9c"}
{"level":"info","ts":"2026-09-18T09:49:28.396Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter-auth","namespace":"test-ns-q2lj2"},"reconcileID":"01fbbe72-5002-4c07-830d-eaeb099c026b"}
{"level":"info","ts":"2026-09-18T09:49:28.412Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-q2lj2"},"reconcileID":"783f0f81-07c1-40fe-ad6a-e262f5f08032"}
{"level":"info","ts":"2026-09-18T09:49:31.173Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b62733e4-f594-43dc-825d-f0792b80eefc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:31.173Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b62733e4-f594-43dc-825d-f0792b80eefc","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:31.173Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b62733e4-f594-43dc-825d-f0792b80eefc","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:31.173Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b62733e4-f594-43dc-825d-f0792b80eefc"}
{"level":"info","ts":"2026-09-18T09:49:31.173Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b62733e4-f594-43dc-825d-f0792b80eefc","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:31.173Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b62733e4-f594-43dc-825d-f0792b80eefc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:34.174Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"1293aaaa-cdf5-4dc6-9bea-2251be39f219","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:34.174Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"1293aaaa-cdf5-4dc6-9bea-2251be39f219","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:34.174Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"1293aaaa-cdf5-4dc6-9bea-2251be39f219","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:34.175Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"1293aaaa-cdf5-4dc6-9bea-2251be39f219"}
{"level":"info","ts":"2026-09-18T09:49:34.175Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"1293aaaa-cdf5-4dc6-9bea-2251be39f219","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:34.175Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"1293aaaa-cdf5-4dc6-9bea-2251be39f219","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:36.346Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7d339ca2-8ca4-48ef-951d-30dde73c641f"}
{"level":"info","ts":"2026-09-18T09:49:36.346Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7d339ca2-8ca4-48ef-951d-30dde73c641f"}
{"level":"info","ts":"2026-09-18T09:49:36.346Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7d339ca2-8ca4-48ef-951d-30dde73c641f"}
{"level":"info","ts":"2026-09-18T09:49:36.346Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7d339ca2-8ca4-48ef-951d-30dde73c641f","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:36.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7d339ca2-8ca4-48ef-951d-30dde73c641f","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:36.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7d339ca2-8ca4-48ef-951d-30dde73c641f","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:36.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7d339ca2-8ca4-48ef-951d-30dde73c641f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:36.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7d339ca2-8ca4-48ef-951d-30dde73c641f","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:36.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7d339ca2-8ca4-48ef-951d-30dde73c641f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:36.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7d339ca2-8ca4-48ef-951d-30dde73c641f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:36.347Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7d339ca2-8ca4-48ef-951d-30dde73c641f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:36.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7d339ca2-8ca4-48ef-951d-30dde73c641f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:36.347Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7d339ca2-8ca4-48ef-951d-30dde73c641f","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:36.347Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7d339ca2-8ca4-48ef-951d-30dde73c641f","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:36.347Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7d339ca2-8ca4-48ef-951d-30dde73c641f","name":"long-running-2qcbf","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:49:36.363Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"ad591d83-fc4a-4ca9-befe-57295ad3c710","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:36.363Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"4d5ec0ba-43c8-492c-b682-5a15a763dd02","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:36.363Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"ad591d83-fc4a-4ca9-befe-57295ad3c710","pod":"long-running-2qcbf-head-cz5d7","ip":"10.132.0.34"}
{"level":"info","ts":"2026-09-18T09:49:36.363Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"ad591d83-fc4a-4ca9-befe-57295ad3c710","pod":"long-running-2qcbf-small-group-worker-94s9h","ip":"10.133.0.32"}
{"level":"info","ts":"2026-09-18T09:49:36.363Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"ad591d83-fc4a-4ca9-befe-57295ad3c710","rayCluster":"long-running-2qcbf","podIPs":["10.132.0.34","10.133.0.32"]}
{"level":"info","ts":"2026-09-18T09:49:36.363Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b45ae9ab-3962-4528-8a5f-503dcf94a1a3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:36.363Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b45ae9ab-3962-4528-8a5f-503dcf94a1a3","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:36.363Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b45ae9ab-3962-4528-8a5f-503dcf94a1a3","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:36.363Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b45ae9ab-3962-4528-8a5f-503dcf94a1a3"}
{"level":"info","ts":"2026-09-18T09:49:36.363Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b45ae9ab-3962-4528-8a5f-503dcf94a1a3","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:36.363Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b45ae9ab-3962-4528-8a5f-503dcf94a1a3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:36.363Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b45ae9ab-3962-4528-8a5f-503dcf94a1a3","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:49:36.383Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"aad7f750-3b1c-4d6c-b6c6-83e389bb414d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:36.383Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"aad7f750-3b1c-4d6c-b6c6-83e389bb414d","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:36.383Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"aad7f750-3b1c-4d6c-b6c6-83e389bb414d","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:36.383Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"aad7f750-3b1c-4d6c-b6c6-83e389bb414d"}
{"level":"info","ts":"2026-09-18T09:49:36.383Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"aad7f750-3b1c-4d6c-b6c6-83e389bb414d","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:36.383Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"aad7f750-3b1c-4d6c-b6c6-83e389bb414d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:36.387Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"4d5ec0ba-43c8-492c-b682-5a15a763dd02","name":"long-running-2qcbf-head"}
{"level":"info","ts":"2026-09-18T09:49:36.400Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"4d5ec0ba-43c8-492c-b682-5a15a763dd02","name":"long-running-2qcbf-workers"}
{"level":"info","ts":"2026-09-18T09:49:36.400Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"4d5ec0ba-43c8-492c-b682-5a15a763dd02","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:37.175Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4a071de2-65ff-4922-8e7e-542fd50e3bf3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:37.175Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4a071de2-65ff-4922-8e7e-542fd50e3bf3","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:37.175Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4a071de2-65ff-4922-8e7e-542fd50e3bf3","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:37.175Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4a071de2-65ff-4922-8e7e-542fd50e3bf3"}
{"level":"info","ts":"2026-09-18T09:49:37.175Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4a071de2-65ff-4922-8e7e-542fd50e3bf3","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:37.175Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4a071de2-65ff-4922-8e7e-542fd50e3bf3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:38.364Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5e303478-27d5-4c2d-a049-869bf9168f3d"}
{"level":"info","ts":"2026-09-18T09:49:38.364Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5e303478-27d5-4c2d-a049-869bf9168f3d"}
{"level":"info","ts":"2026-09-18T09:49:38.364Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5e303478-27d5-4c2d-a049-869bf9168f3d"}
{"level":"info","ts":"2026-09-18T09:49:38.364Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5e303478-27d5-4c2d-a049-869bf9168f3d","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:49:38.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5e303478-27d5-4c2d-a049-869bf9168f3d","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:49:38.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5e303478-27d5-4c2d-a049-869bf9168f3d","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:38.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5e303478-27d5-4c2d-a049-869bf9168f3d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:49:38.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5e303478-27d5-4c2d-a049-869bf9168f3d","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:49:38.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5e303478-27d5-4c2d-a049-869bf9168f3d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:38.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5e303478-27d5-4c2d-a049-869bf9168f3d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:49:38.364Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5e303478-27d5-4c2d-a049-869bf9168f3d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:49:38.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5e303478-27d5-4c2d-a049-869bf9168f3d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:49:38.364Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5e303478-27d5-4c2d-a049-869bf9168f3d","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:38.364Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5e303478-27d5-4c2d-a049-869bf9168f3d","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:49:38.364Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5e303478-27d5-4c2d-a049-869bf9168f3d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:49:38.364Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5e303478-27d5-4c2d-a049-869bf9168f3d","seconds":300}
{"level":"info","ts":"2026-09-18T09:49:40.176Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c35da74f-70a3-48e4-a35d-c47ba97be81f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:40.176Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c35da74f-70a3-48e4-a35d-c47ba97be81f","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:40.176Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c35da74f-70a3-48e4-a35d-c47ba97be81f","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:40.176Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c35da74f-70a3-48e4-a35d-c47ba97be81f"}
{"level":"info","ts":"2026-09-18T09:49:40.176Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c35da74f-70a3-48e4-a35d-c47ba97be81f","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:40.176Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c35da74f-70a3-48e4-a35d-c47ba97be81f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:43.177Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c0df97a5-1f44-4b98-a443-4ae9b1e8df56","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:43.177Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c0df97a5-1f44-4b98-a443-4ae9b1e8df56","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:43.177Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c0df97a5-1f44-4b98-a443-4ae9b1e8df56","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:43.177Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c0df97a5-1f44-4b98-a443-4ae9b1e8df56"}
{"level":"info","ts":"2026-09-18T09:49:43.177Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c0df97a5-1f44-4b98-a443-4ae9b1e8df56","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:43.177Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c0df97a5-1f44-4b98-a443-4ae9b1e8df56","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:43.495Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"d584d3ed-aaf1-45b1-b937-9977d39e4a46","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:43.495Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"d584d3ed-aaf1-45b1-b937-9977d39e4a46","pod":"long-running-2qcbf-head-cz5d7","ip":"10.132.0.34"}
{"level":"info","ts":"2026-09-18T09:49:43.495Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"d584d3ed-aaf1-45b1-b937-9977d39e4a46","pod":"long-running-2qcbf-small-group-worker-94s9h","ip":"10.133.0.32"}
{"level":"info","ts":"2026-09-18T09:49:43.495Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"d584d3ed-aaf1-45b1-b937-9977d39e4a46","rayCluster":"long-running-2qcbf","podIPs":["10.132.0.34","10.133.0.32"]}
{"level":"info","ts":"2026-09-18T09:49:46.178Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d95904a8-b1b5-421b-ba60-377d8cf49809","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:46.178Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d95904a8-b1b5-421b-ba60-377d8cf49809","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:46.178Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d95904a8-b1b5-421b-ba60-377d8cf49809","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:46.178Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d95904a8-b1b5-421b-ba60-377d8cf49809"}
{"level":"info","ts":"2026-09-18T09:49:46.178Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d95904a8-b1b5-421b-ba60-377d8cf49809","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:46.178Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d95904a8-b1b5-421b-ba60-377d8cf49809","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:49.179Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"84cf0cde-13af-4358-887c-6d5bf601331d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:49.179Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"84cf0cde-13af-4358-887c-6d5bf601331d","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:49.179Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"84cf0cde-13af-4358-887c-6d5bf601331d","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:49.179Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"84cf0cde-13af-4358-887c-6d5bf601331d"}
{"level":"info","ts":"2026-09-18T09:49:49.179Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"84cf0cde-13af-4358-887c-6d5bf601331d","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:49.179Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"84cf0cde-13af-4358-887c-6d5bf601331d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:52.180Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"95c9475d-4384-4480-bdc4-2b47ef193012","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:52.180Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"95c9475d-4384-4480-bdc4-2b47ef193012","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:52.180Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"95c9475d-4384-4480-bdc4-2b47ef193012","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:52.180Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"95c9475d-4384-4480-bdc4-2b47ef193012"}
{"level":"info","ts":"2026-09-18T09:49:52.180Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"95c9475d-4384-4480-bdc4-2b47ef193012","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:52.180Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"95c9475d-4384-4480-bdc4-2b47ef193012","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:55.181Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb9e2fd1-10c5-41e7-b02f-9ddb99825641","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:55.181Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb9e2fd1-10c5-41e7-b02f-9ddb99825641","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:55.181Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb9e2fd1-10c5-41e7-b02f-9ddb99825641","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:55.181Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb9e2fd1-10c5-41e7-b02f-9ddb99825641"}
{"level":"info","ts":"2026-09-18T09:49:55.181Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb9e2fd1-10c5-41e7-b02f-9ddb99825641","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:55.181Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb9e2fd1-10c5-41e7-b02f-9ddb99825641","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:49:58.182Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"20ea48ae-3273-4f64-91cb-558ba34737f2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:49:58.182Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"20ea48ae-3273-4f64-91cb-558ba34737f2","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:49:58.182Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"20ea48ae-3273-4f64-91cb-558ba34737f2","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:49:58.182Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"20ea48ae-3273-4f64-91cb-558ba34737f2"}
{"level":"info","ts":"2026-09-18T09:49:58.182Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"20ea48ae-3273-4f64-91cb-558ba34737f2","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:49:58.182Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"20ea48ae-3273-4f64-91cb-558ba34737f2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:01.183Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"fc9b5812-ffff-4f7b-a17a-3638c6f38218","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:01.183Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"fc9b5812-ffff-4f7b-a17a-3638c6f38218","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:01.183Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"fc9b5812-ffff-4f7b-a17a-3638c6f38218","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:01.183Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"fc9b5812-ffff-4f7b-a17a-3638c6f38218"}
{"level":"info","ts":"2026-09-18T09:50:01.183Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"fc9b5812-ffff-4f7b-a17a-3638c6f38218","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:01.183Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"fc9b5812-ffff-4f7b-a17a-3638c6f38218","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:04.184Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"392720cb-8a8e-4152-9cf1-72e4f530c56b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:04.184Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"392720cb-8a8e-4152-9cf1-72e4f530c56b","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:04.184Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"392720cb-8a8e-4152-9cf1-72e4f530c56b","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:04.185Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"392720cb-8a8e-4152-9cf1-72e4f530c56b"}
{"level":"info","ts":"2026-09-18T09:50:04.185Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"392720cb-8a8e-4152-9cf1-72e4f530c56b","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:04.185Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"392720cb-8a8e-4152-9cf1-72e4f530c56b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:07.185Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"54ba87a1-d727-4a86-8d53-78c9c397cb83","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:07.185Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"54ba87a1-d727-4a86-8d53-78c9c397cb83","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:07.185Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"54ba87a1-d727-4a86-8d53-78c9c397cb83","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:07.186Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"54ba87a1-d727-4a86-8d53-78c9c397cb83"}
{"level":"info","ts":"2026-09-18T09:50:07.186Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"54ba87a1-d727-4a86-8d53-78c9c397cb83","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:07.186Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"54ba87a1-d727-4a86-8d53-78c9c397cb83","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:10.187Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e910eea5-b954-4915-94c7-5b82a4d513fc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:10.187Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e910eea5-b954-4915-94c7-5b82a4d513fc","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:10.187Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e910eea5-b954-4915-94c7-5b82a4d513fc","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:10.187Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e910eea5-b954-4915-94c7-5b82a4d513fc"}
{"level":"info","ts":"2026-09-18T09:50:10.187Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e910eea5-b954-4915-94c7-5b82a4d513fc","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:10.187Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e910eea5-b954-4915-94c7-5b82a4d513fc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:13.188Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ff30c9ad-e8df-4420-aeca-b5bd98f26d8f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:13.188Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ff30c9ad-e8df-4420-aeca-b5bd98f26d8f","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:13.188Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ff30c9ad-e8df-4420-aeca-b5bd98f26d8f","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:13.188Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ff30c9ad-e8df-4420-aeca-b5bd98f26d8f"}
{"level":"info","ts":"2026-09-18T09:50:13.188Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ff30c9ad-e8df-4420-aeca-b5bd98f26d8f","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:13.188Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ff30c9ad-e8df-4420-aeca-b5bd98f26d8f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:16.188Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"76192813-26e8-4b97-97c1-ad90cb362a33","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:16.188Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"76192813-26e8-4b97-97c1-ad90cb362a33","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:16.188Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"76192813-26e8-4b97-97c1-ad90cb362a33","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:16.189Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"76192813-26e8-4b97-97c1-ad90cb362a33"}
{"level":"info","ts":"2026-09-18T09:50:16.189Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"76192813-26e8-4b97-97c1-ad90cb362a33","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:16.189Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"76192813-26e8-4b97-97c1-ad90cb362a33","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:19.190Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"979ffd18-1eb5-4d95-9676-46730d297825","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:19.190Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"979ffd18-1eb5-4d95-9676-46730d297825","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:19.190Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"979ffd18-1eb5-4d95-9676-46730d297825","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:19.190Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"979ffd18-1eb5-4d95-9676-46730d297825"}
{"level":"info","ts":"2026-09-18T09:50:19.190Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"979ffd18-1eb5-4d95-9676-46730d297825","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:19.190Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"979ffd18-1eb5-4d95-9676-46730d297825","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:22.190Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ad667b52-43e7-4c0b-98af-93acbeb10d1e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:22.191Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ad667b52-43e7-4c0b-98af-93acbeb10d1e","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:22.191Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ad667b52-43e7-4c0b-98af-93acbeb10d1e","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:22.191Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ad667b52-43e7-4c0b-98af-93acbeb10d1e"}
{"level":"info","ts":"2026-09-18T09:50:22.191Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ad667b52-43e7-4c0b-98af-93acbeb10d1e","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:22.191Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ad667b52-43e7-4c0b-98af-93acbeb10d1e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:25.192Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2dfcab9e-814c-4090-9a3e-bf9808f1ba83","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:25.192Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2dfcab9e-814c-4090-9a3e-bf9808f1ba83","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:25.192Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2dfcab9e-814c-4090-9a3e-bf9808f1ba83","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:25.192Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2dfcab9e-814c-4090-9a3e-bf9808f1ba83"}
{"level":"info","ts":"2026-09-18T09:50:25.192Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2dfcab9e-814c-4090-9a3e-bf9808f1ba83","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:25.192Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2dfcab9e-814c-4090-9a3e-bf9808f1ba83","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:28.192Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"dda8aa8b-db2c-42ad-b3f0-6408bfca93d5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:28.192Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"dda8aa8b-db2c-42ad-b3f0-6408bfca93d5","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:28.192Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"dda8aa8b-db2c-42ad-b3f0-6408bfca93d5","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:28.192Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"dda8aa8b-db2c-42ad-b3f0-6408bfca93d5"}
{"level":"info","ts":"2026-09-18T09:50:28.192Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"dda8aa8b-db2c-42ad-b3f0-6408bfca93d5","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:28.192Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"dda8aa8b-db2c-42ad-b3f0-6408bfca93d5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:30.351Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"fc2e4d58-6106-4de1-98c7-7589a054a3da"}
{"level":"info","ts":"2026-09-18T09:50:30.351Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"fc2e4d58-6106-4de1-98c7-7589a054a3da"}
{"level":"info","ts":"2026-09-18T09:50:30.351Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"fc2e4d58-6106-4de1-98c7-7589a054a3da"}
{"level":"info","ts":"2026-09-18T09:50:30.351Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"fc2e4d58-6106-4de1-98c7-7589a054a3da","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:30.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"fc2e4d58-6106-4de1-98c7-7589a054a3da","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:50:30.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"fc2e4d58-6106-4de1-98c7-7589a054a3da","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:30.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"fc2e4d58-6106-4de1-98c7-7589a054a3da","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:50:30.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"fc2e4d58-6106-4de1-98c7-7589a054a3da","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:30.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"fc2e4d58-6106-4de1-98c7-7589a054a3da","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:30.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"fc2e4d58-6106-4de1-98c7-7589a054a3da","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:50:30.351Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"fc2e4d58-6106-4de1-98c7-7589a054a3da","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:50:30.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"fc2e4d58-6106-4de1-98c7-7589a054a3da","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:30.352Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"fc2e4d58-6106-4de1-98c7-7589a054a3da","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:30.352Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"fc2e4d58-6106-4de1-98c7-7589a054a3da","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:30.352Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"fc2e4d58-6106-4de1-98c7-7589a054a3da","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:50:30.352Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"fc2e4d58-6106-4de1-98c7-7589a054a3da","seconds":300}
{"level":"info","ts":"2026-09-18T09:50:31.193Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"3080173e-0bff-48ac-8dde-09d54394b32e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:31.193Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"3080173e-0bff-48ac-8dde-09d54394b32e","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:31.193Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"3080173e-0bff-48ac-8dde-09d54394b32e","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:31.194Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"3080173e-0bff-48ac-8dde-09d54394b32e"}
{"level":"info","ts":"2026-09-18T09:50:31.194Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"3080173e-0bff-48ac-8dde-09d54394b32e","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:31.194Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"3080173e-0bff-48ac-8dde-09d54394b32e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:31.342Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40febbd8-4bd3-459d-9b0b-93ef5ba00cf2"}
{"level":"info","ts":"2026-09-18T09:50:31.342Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40febbd8-4bd3-459d-9b0b-93ef5ba00cf2"}
{"level":"info","ts":"2026-09-18T09:50:31.342Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40febbd8-4bd3-459d-9b0b-93ef5ba00cf2"}
{"level":"info","ts":"2026-09-18T09:50:31.342Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40febbd8-4bd3-459d-9b0b-93ef5ba00cf2","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:31.342Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40febbd8-4bd3-459d-9b0b-93ef5ba00cf2","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:50:31.342Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40febbd8-4bd3-459d-9b0b-93ef5ba00cf2","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:31.342Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40febbd8-4bd3-459d-9b0b-93ef5ba00cf2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:50:31.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40febbd8-4bd3-459d-9b0b-93ef5ba00cf2","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:31.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40febbd8-4bd3-459d-9b0b-93ef5ba00cf2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:31.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40febbd8-4bd3-459d-9b0b-93ef5ba00cf2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:50:31.343Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40febbd8-4bd3-459d-9b0b-93ef5ba00cf2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:50:31.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40febbd8-4bd3-459d-9b0b-93ef5ba00cf2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:31.343Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40febbd8-4bd3-459d-9b0b-93ef5ba00cf2","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:31.343Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40febbd8-4bd3-459d-9b0b-93ef5ba00cf2","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:31.343Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40febbd8-4bd3-459d-9b0b-93ef5ba00cf2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:50:31.343Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"40febbd8-4bd3-459d-9b0b-93ef5ba00cf2","seconds":300}
{"level":"info","ts":"2026-09-18T09:50:34.194Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a8e75488-c0da-4d83-9c4e-14a7ae4b566a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:34.194Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a8e75488-c0da-4d83-9c4e-14a7ae4b566a","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:34.194Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a8e75488-c0da-4d83-9c4e-14a7ae4b566a","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:34.194Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a8e75488-c0da-4d83-9c4e-14a7ae4b566a"}
{"level":"info","ts":"2026-09-18T09:50:34.194Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a8e75488-c0da-4d83-9c4e-14a7ae4b566a","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:34.194Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a8e75488-c0da-4d83-9c4e-14a7ae4b566a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:37.195Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"53cc6855-14f4-45f2-b55c-23b4930acc46","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:37.195Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"53cc6855-14f4-45f2-b55c-23b4930acc46","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:37.195Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"53cc6855-14f4-45f2-b55c-23b4930acc46","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:37.195Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"53cc6855-14f4-45f2-b55c-23b4930acc46"}
{"level":"info","ts":"2026-09-18T09:50:37.195Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"53cc6855-14f4-45f2-b55c-23b4930acc46","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:37.195Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"53cc6855-14f4-45f2-b55c-23b4930acc46","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:40.197Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7cf5dfb5-d467-4ff2-8a92-e5ab9c96ebdb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:40.197Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7cf5dfb5-d467-4ff2-8a92-e5ab9c96ebdb","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:40.197Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7cf5dfb5-d467-4ff2-8a92-e5ab9c96ebdb","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:40.197Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7cf5dfb5-d467-4ff2-8a92-e5ab9c96ebdb"}
{"level":"info","ts":"2026-09-18T09:50:40.197Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7cf5dfb5-d467-4ff2-8a92-e5ab9c96ebdb","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:40.197Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7cf5dfb5-d467-4ff2-8a92-e5ab9c96ebdb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:40.379Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f614b8c3-e154-4529-aeba-84fcc0550d01"}
{"level":"info","ts":"2026-09-18T09:50:40.379Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f614b8c3-e154-4529-aeba-84fcc0550d01"}
{"level":"info","ts":"2026-09-18T09:50:40.379Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f614b8c3-e154-4529-aeba-84fcc0550d01"}
{"level":"info","ts":"2026-09-18T09:50:40.379Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f614b8c3-e154-4529-aeba-84fcc0550d01","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:40.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f614b8c3-e154-4529-aeba-84fcc0550d01","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:50:40.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f614b8c3-e154-4529-aeba-84fcc0550d01","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:40.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f614b8c3-e154-4529-aeba-84fcc0550d01","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:50:40.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f614b8c3-e154-4529-aeba-84fcc0550d01","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:40.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f614b8c3-e154-4529-aeba-84fcc0550d01","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:40.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f614b8c3-e154-4529-aeba-84fcc0550d01","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:50:40.379Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f614b8c3-e154-4529-aeba-84fcc0550d01","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:50:40.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f614b8c3-e154-4529-aeba-84fcc0550d01","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:40.380Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f614b8c3-e154-4529-aeba-84fcc0550d01","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:40.380Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f614b8c3-e154-4529-aeba-84fcc0550d01","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:40.380Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f614b8c3-e154-4529-aeba-84fcc0550d01","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:50:40.380Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"f614b8c3-e154-4529-aeba-84fcc0550d01","seconds":300}
{"level":"info","ts":"2026-09-18T09:50:41.388Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7a293023-1f1c-4190-95c6-93a9ed5a8bd9"}
{"level":"info","ts":"2026-09-18T09:50:41.388Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7a293023-1f1c-4190-95c6-93a9ed5a8bd9"}
{"level":"info","ts":"2026-09-18T09:50:41.388Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7a293023-1f1c-4190-95c6-93a9ed5a8bd9"}
{"level":"info","ts":"2026-09-18T09:50:41.389Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7a293023-1f1c-4190-95c6-93a9ed5a8bd9","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:41.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7a293023-1f1c-4190-95c6-93a9ed5a8bd9","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:50:41.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7a293023-1f1c-4190-95c6-93a9ed5a8bd9","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:41.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7a293023-1f1c-4190-95c6-93a9ed5a8bd9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:50:41.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7a293023-1f1c-4190-95c6-93a9ed5a8bd9","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:41.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7a293023-1f1c-4190-95c6-93a9ed5a8bd9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:41.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7a293023-1f1c-4190-95c6-93a9ed5a8bd9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:50:41.389Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7a293023-1f1c-4190-95c6-93a9ed5a8bd9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:50:41.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7a293023-1f1c-4190-95c6-93a9ed5a8bd9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:41.389Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7a293023-1f1c-4190-95c6-93a9ed5a8bd9","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:49:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:50:41.389Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7a293023-1f1c-4190-95c6-93a9ed5a8bd9","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:49:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:50:41.389Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"7a293023-1f1c-4190-95c6-93a9ed5a8bd9","name":"long-running-2qcbf","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:50:41.405Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"32dff4fa-aabf-46bd-b32d-c58cfaa44c85","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:41.405Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"a3f44519-0ede-40ed-9f9a-b3fe0ca9a46a","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:41.405Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"8845298d-0cd8-41d6-ad45-94914448b48e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:41.405Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"8845298d-0cd8-41d6-ad45-94914448b48e","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:41.405Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"8845298d-0cd8-41d6-ad45-94914448b48e","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:41.405Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"a3f44519-0ede-40ed-9f9a-b3fe0ca9a46a","pod":"long-running-2qcbf-small-group-worker-94s9h","ip":"10.133.0.32"}
{"level":"info","ts":"2026-09-18T09:50:41.405Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"a3f44519-0ede-40ed-9f9a-b3fe0ca9a46a","pod":"long-running-2qcbf-head-cz5d7","ip":"10.132.0.34"}
{"level":"info","ts":"2026-09-18T09:50:41.405Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"a3f44519-0ede-40ed-9f9a-b3fe0ca9a46a","rayCluster":"long-running-2qcbf","podIPs":["10.133.0.32","10.132.0.34"]}
{"level":"info","ts":"2026-09-18T09:50:41.405Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"8845298d-0cd8-41d6-ad45-94914448b48e"}
{"level":"info","ts":"2026-09-18T09:50:41.405Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"8845298d-0cd8-41d6-ad45-94914448b48e","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:41.405Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"8845298d-0cd8-41d6-ad45-94914448b48e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:41.405Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"8845298d-0cd8-41d6-ad45-94914448b48e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:50:41.422Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"8d8a541d-629d-47f4-a76f-59ad5c3b5a2e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:41.422Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"8d8a541d-629d-47f4-a76f-59ad5c3b5a2e","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:41.422Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"8d8a541d-629d-47f4-a76f-59ad5c3b5a2e","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:41.422Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"8d8a541d-629d-47f4-a76f-59ad5c3b5a2e"}
{"level":"info","ts":"2026-09-18T09:50:41.422Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"8d8a541d-629d-47f4-a76f-59ad5c3b5a2e","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:41.422Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"8d8a541d-629d-47f4-a76f-59ad5c3b5a2e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:41.431Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"32dff4fa-aabf-46bd-b32d-c58cfaa44c85","name":"long-running-2qcbf-head"}
{"level":"info","ts":"2026-09-18T09:50:41.443Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"32dff4fa-aabf-46bd-b32d-c58cfaa44c85","name":"long-running-2qcbf-workers"}
{"level":"info","ts":"2026-09-18T09:50:41.443Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"32dff4fa-aabf-46bd-b32d-c58cfaa44c85","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:43.198Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a58a4ef4-710f-465f-9410-0e01897bbd92","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:43.198Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a58a4ef4-710f-465f-9410-0e01897bbd92","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:43.198Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a58a4ef4-710f-465f-9410-0e01897bbd92","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:43.198Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a58a4ef4-710f-465f-9410-0e01897bbd92"}
{"level":"info","ts":"2026-09-18T09:50:43.198Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a58a4ef4-710f-465f-9410-0e01897bbd92","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:43.198Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"a58a4ef4-710f-465f-9410-0e01897bbd92","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:43.406Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"58baf96b-3187-4500-8492-a7c6ec13f1af"}
{"level":"info","ts":"2026-09-18T09:50:43.406Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"58baf96b-3187-4500-8492-a7c6ec13f1af"}
{"level":"info","ts":"2026-09-18T09:50:43.406Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"58baf96b-3187-4500-8492-a7c6ec13f1af"}
{"level":"info","ts":"2026-09-18T09:50:43.406Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"58baf96b-3187-4500-8492-a7c6ec13f1af","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:43.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"58baf96b-3187-4500-8492-a7c6ec13f1af","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:50:43.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"58baf96b-3187-4500-8492-a7c6ec13f1af","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:43.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"58baf96b-3187-4500-8492-a7c6ec13f1af","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:50:43.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"58baf96b-3187-4500-8492-a7c6ec13f1af","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:43.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"58baf96b-3187-4500-8492-a7c6ec13f1af","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:43.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"58baf96b-3187-4500-8492-a7c6ec13f1af","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:50:43.406Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"58baf96b-3187-4500-8492-a7c6ec13f1af","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:50:43.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"58baf96b-3187-4500-8492-a7c6ec13f1af","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:43.406Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"58baf96b-3187-4500-8492-a7c6ec13f1af","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:49:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:50:43.406Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"58baf96b-3187-4500-8492-a7c6ec13f1af","podName":"long-running-2qcbf-small-group-worker-94s9h","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:49:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:50:43.406Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"58baf96b-3187-4500-8492-a7c6ec13f1af","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:50:43.406Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"58baf96b-3187-4500-8492-a7c6ec13f1af","seconds":300}
{"level":"info","ts":"2026-09-18T09:50:43.495Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"b4c643e2-b13a-4502-9da5-20d7d25a78e8","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:43.495Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"b4c643e2-b13a-4502-9da5-20d7d25a78e8","pod":"long-running-2qcbf-head-cz5d7","ip":"10.132.0.34"}
{"level":"info","ts":"2026-09-18T09:50:43.495Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"b4c643e2-b13a-4502-9da5-20d7d25a78e8","pod":"long-running-2qcbf-small-group-worker-94s9h","ip":"10.133.0.32"}
{"level":"info","ts":"2026-09-18T09:50:43.495Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"b4c643e2-b13a-4502-9da5-20d7d25a78e8","rayCluster":"long-running-2qcbf","podIPs":["10.132.0.34","10.133.0.32"]}
{"level":"info","ts":"2026-09-18T09:50:46.199Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"833a0eb5-3599-44ed-befc-defcd4826949","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:46.199Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"833a0eb5-3599-44ed-befc-defcd4826949","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:46.199Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"833a0eb5-3599-44ed-befc-defcd4826949","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:46.199Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"833a0eb5-3599-44ed-befc-defcd4826949"}
{"level":"info","ts":"2026-09-18T09:50:46.199Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"833a0eb5-3599-44ed-befc-defcd4826949","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:46.199Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"833a0eb5-3599-44ed-befc-defcd4826949","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:49.201Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"f96fd6c1-f7b1-4f64-bd5c-49e2b6989c6d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:49.201Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"f96fd6c1-f7b1-4f64-bd5c-49e2b6989c6d","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:49.201Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"f96fd6c1-f7b1-4f64-bd5c-49e2b6989c6d","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:49.201Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"f96fd6c1-f7b1-4f64-bd5c-49e2b6989c6d"}
{"level":"info","ts":"2026-09-18T09:50:49.201Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"f96fd6c1-f7b1-4f64-bd5c-49e2b6989c6d","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:49.201Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"f96fd6c1-f7b1-4f64-bd5c-49e2b6989c6d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:52.202Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72470ebb-debf-424e-9ef0-95fc1b92227d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:52.202Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72470ebb-debf-424e-9ef0-95fc1b92227d","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:52.202Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72470ebb-debf-424e-9ef0-95fc1b92227d","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:52.202Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72470ebb-debf-424e-9ef0-95fc1b92227d"}
{"level":"info","ts":"2026-09-18T09:50:52.202Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72470ebb-debf-424e-9ef0-95fc1b92227d","RayCluster":"long-running-2qcbf","State":""}
{"level":"info","ts":"2026-09-18T09:50:52.202Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72470ebb-debf-424e-9ef0-95fc1b92227d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:52.411Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6bd93d6f-51e5-4dff-8a44-a741326e77a1"}
{"level":"info","ts":"2026-09-18T09:50:52.411Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6bd93d6f-51e5-4dff-8a44-a741326e77a1"}
{"level":"info","ts":"2026-09-18T09:50:52.411Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6bd93d6f-51e5-4dff-8a44-a741326e77a1"}
{"level":"info","ts":"2026-09-18T09:50:52.411Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6bd93d6f-51e5-4dff-8a44-a741326e77a1","1 head service found":"long-running-2qcbf-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:52.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6bd93d6f-51e5-4dff-8a44-a741326e77a1","Found 1 head Pod":"long-running-2qcbf-head-cz5d7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:50:52.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6bd93d6f-51e5-4dff-8a44-a741326e77a1","head Pod":"long-running-2qcbf-head-cz5d7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2qcbf-head-cz5d7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:52.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6bd93d6f-51e5-4dff-8a44-a741326e77a1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:50:52.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6bd93d6f-51e5-4dff-8a44-a741326e77a1","worker Pod":"long-running-2qcbf-small-group-worker-94s9h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2qcbf-small-group-worker-94s9h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:52.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6bd93d6f-51e5-4dff-8a44-a741326e77a1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:52.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6bd93d6f-51e5-4dff-8a44-a741326e77a1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:50:52.411Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6bd93d6f-51e5-4dff-8a44-a741326e77a1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:50:52.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6bd93d6f-51e5-4dff-8a44-a741326e77a1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:52.411Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"6bd93d6f-51e5-4dff-8a44-a741326e77a1","name":"long-running-2qcbf","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:52Z","stateTransitionTimes":{"ready":"2026-09-18T09:50:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:50:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:50:52.426Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"d8bda1d0-61c6-4767-b0d2-0aa920525dca","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:52.426Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"d8bda1d0-61c6-4767-b0d2-0aa920525dca","pod":"long-running-2qcbf-head-cz5d7","ip":"10.132.0.34"}
{"level":"info","ts":"2026-09-18T09:50:52.426Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"d8bda1d0-61c6-4767-b0d2-0aa920525dca","pod":"long-running-2qcbf-small-group-worker-94s9h","ip":"10.133.0.32"}
{"level":"info","ts":"2026-09-18T09:50:52.426Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"d8bda1d0-61c6-4767-b0d2-0aa920525dca","rayCluster":"long-running-2qcbf","podIPs":["10.132.0.34","10.133.0.32"]}
{"level":"info","ts":"2026-09-18T09:50:52.426Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"591c5956-02ba-46c1-b226-dd566bf6ec1b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:52.426Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"591c5956-02ba-46c1-b226-dd566bf6ec1b","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:52.426Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"c54f0eda-a9a4-42d6-b26e-07d1a1e8b851","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:52.426Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"591c5956-02ba-46c1-b226-dd566bf6ec1b","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:52.426Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"591c5956-02ba-46c1-b226-dd566bf6ec1b"}
{"level":"info","ts":"2026-09-18T09:50:52.436Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"591c5956-02ba-46c1-b226-dd566bf6ec1b","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-09-18T09:50:52.436Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"591c5956-02ba-46c1-b226-dd566bf6ec1b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","dashboardURL":"long-running-2qcbf-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:52.436Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"591c5956-02ba-46c1-b226-dd566bf6ec1b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:50:52.441Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"c54f0eda-a9a4-42d6-b26e-07d1a1e8b851","name":"long-running-2qcbf-head"}
{"level":"info","ts":"2026-09-18T09:50:52.450Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"96ff6258-77e0-4baa-b6fd-8e8899cf1fb3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:52.450Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"96ff6258-77e0-4baa-b6fd-8e8899cf1fb3","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:52.450Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"96ff6258-77e0-4baa-b6fd-8e8899cf1fb3","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:52.450Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"96ff6258-77e0-4baa-b6fd-8e8899cf1fb3"}
{"level":"info","ts":"2026-09-18T09:50:52.450Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"96ff6258-77e0-4baa-b6fd-8e8899cf1fb3","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-09-18T09:50:52.450Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"96ff6258-77e0-4baa-b6fd-8e8899cf1fb3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","dashboardURL":"long-running-2qcbf-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:52.450Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"96ff6258-77e0-4baa-b6fd-8e8899cf1fb3","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:50:52.457Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"c54f0eda-a9a4-42d6-b26e-07d1a1e8b851","name":"long-running-2qcbf-workers"}
{"level":"info","ts":"2026-09-18T09:50:52.457Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"c54f0eda-a9a4-42d6-b26e-07d1a1e8b851","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:52.461Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"96ff6258-77e0-4baa-b6fd-8e8899cf1fb3","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T09:50:52.461Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"96ff6258-77e0-4baa-b6fd-8e8899cf1fb3"}
{"level":"error","ts":"2026-09-18T09:50:52.461Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"96ff6258-77e0-4baa-b6fd-8e8899cf1fb3","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:52.461Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c74c3454-05b7-430a-bbae-53a54ca589b5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:52.461Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c74c3454-05b7-430a-bbae-53a54ca589b5","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:52.462Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c74c3454-05b7-430a-bbae-53a54ca589b5","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:52.462Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c74c3454-05b7-430a-bbae-53a54ca589b5"}
{"level":"info","ts":"2026-09-18T09:50:52.462Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c74c3454-05b7-430a-bbae-53a54ca589b5","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:50:52.478Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c74c3454-05b7-430a-bbae-53a54ca589b5","JobId":"long-running-srdwt","error":"Job long-running-srdwt does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:52.478Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c74c3454-05b7-430a-bbae-53a54ca589b5"}
{"level":"error","ts":"2026-09-18T09:50:52.478Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c74c3454-05b7-430a-bbae-53a54ca589b5","error":"Job long-running-srdwt does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:52.478Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2fa4503f-98e2-4bec-8d26-05fb6344f4dd","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:52.478Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2fa4503f-98e2-4bec-8d26-05fb6344f4dd","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:52.478Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2fa4503f-98e2-4bec-8d26-05fb6344f4dd","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:52.478Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2fa4503f-98e2-4bec-8d26-05fb6344f4dd"}
{"level":"info","ts":"2026-09-18T09:50:52.478Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2fa4503f-98e2-4bec-8d26-05fb6344f4dd","rayCluster":"long-running-2qcbf","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:50:52.483Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2fa4503f-98e2-4bec-8d26-05fb6344f4dd","JobId":"long-running-srdwt","error":"Job long-running-srdwt does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:52.483Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2fa4503f-98e2-4bec-8d26-05fb6344f4dd"}
{"level":"error","ts":"2026-09-18T09:50:52.483Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2fa4503f-98e2-4bec-8d26-05fb6344f4dd","error":"Job long-running-srdwt does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:52.489Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"942e98ee-7060-4a25-baae-ed95d9d340be","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:52.489Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"942e98ee-7060-4a25-baae-ed95d9d340be","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:52.489Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"942e98ee-7060-4a25-baae-ed95d9d340be","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:52.489Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"942e98ee-7060-4a25-baae-ed95d9d340be"}
{"level":"info","ts":"2026-09-18T09:50:52.489Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"942e98ee-7060-4a25-baae-ed95d9d340be","rayCluster":"long-running-2qcbf","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:50:52.493Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"942e98ee-7060-4a25-baae-ed95d9d340be","JobId":"long-running-srdwt","error":"Job long-running-srdwt does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:52.494Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"942e98ee-7060-4a25-baae-ed95d9d340be"}
{"level":"error","ts":"2026-09-18T09:50:52.494Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"942e98ee-7060-4a25-baae-ed95d9d340be","error":"Job long-running-srdwt does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:52.534Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d7bc49e5-2980-4ddf-8034-f3231dc4e9cf","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:52.534Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d7bc49e5-2980-4ddf-8034-f3231dc4e9cf","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:52.534Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d7bc49e5-2980-4ddf-8034-f3231dc4e9cf","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:52.534Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d7bc49e5-2980-4ddf-8034-f3231dc4e9cf"}
{"level":"info","ts":"2026-09-18T09:50:52.534Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d7bc49e5-2980-4ddf-8034-f3231dc4e9cf","rayCluster":"long-running-2qcbf","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:50:52.538Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d7bc49e5-2980-4ddf-8034-f3231dc4e9cf","JobId":"long-running-srdwt","error":"Job long-running-srdwt does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:52.538Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d7bc49e5-2980-4ddf-8034-f3231dc4e9cf"}
{"level":"error","ts":"2026-09-18T09:50:52.538Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d7bc49e5-2980-4ddf-8034-f3231dc4e9cf","error":"Job long-running-srdwt does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:52.619Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b66a7161-b9ff-400b-b06d-f2d3fafa05f9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:52.619Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b66a7161-b9ff-400b-b06d-f2d3fafa05f9","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:52.619Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b66a7161-b9ff-400b-b06d-f2d3fafa05f9","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:52.619Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b66a7161-b9ff-400b-b06d-f2d3fafa05f9"}
{"level":"info","ts":"2026-09-18T09:50:52.619Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b66a7161-b9ff-400b-b06d-f2d3fafa05f9","rayCluster":"long-running-2qcbf","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:50:52.623Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b66a7161-b9ff-400b-b06d-f2d3fafa05f9","JobId":"long-running-srdwt","error":"Job long-running-srdwt does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:52.623Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b66a7161-b9ff-400b-b06d-f2d3fafa05f9"}
{"level":"error","ts":"2026-09-18T09:50:52.623Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b66a7161-b9ff-400b-b06d-f2d3fafa05f9","error":"Job long-running-srdwt does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:52.784Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7792200a-0e4f-4deb-b98e-b46edc76999d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:52.784Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7792200a-0e4f-4deb-b98e-b46edc76999d","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:52.784Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7792200a-0e4f-4deb-b98e-b46edc76999d","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:52.784Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7792200a-0e4f-4deb-b98e-b46edc76999d"}
{"level":"info","ts":"2026-09-18T09:50:52.784Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7792200a-0e4f-4deb-b98e-b46edc76999d","rayCluster":"long-running-2qcbf","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:50:52.790Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7792200a-0e4f-4deb-b98e-b46edc76999d","JobId":"long-running-srdwt","error":"Job long-running-srdwt does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:52.790Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7792200a-0e4f-4deb-b98e-b46edc76999d"}
{"level":"error","ts":"2026-09-18T09:50:52.790Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7792200a-0e4f-4deb-b98e-b46edc76999d","error":"Job long-running-srdwt does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:52.808Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4a07a3c8-e017-4660-8f20-d279bf376e82","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:52.808Z","logger":"controllers.RayJob","msg":"Try to transition the status to `Suspending`","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4a07a3c8-e017-4660-8f20-d279bf376e82","oldStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:50:52.808Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4a07a3c8-e017-4660-8f20-d279bf376e82","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","dashboardURL":"long-running-2qcbf-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","dashboardURL":"long-running-2qcbf-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobDeploymentStatus":"Suspending","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:52.808Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4a07a3c8-e017-4660-8f20-d279bf376e82","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Suspending"}
{"level":"info","ts":"2026-09-18T09:50:52.830Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"35438a2f-ee25-4a76-a22b-c9c3467944cf","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:52.843Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"ef6ee6ff-0671-4ef2-9a63-aa78870f1d04"}
{"level":"info","ts":"2026-09-18T09:50:52.844Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"1e0b386d-b99a-4078-af31-bf139da28dea","namespacedName":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:52.844Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"1e0b386d-b99a-4078-af31-bf139da28dea","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:52.844Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"1e0b386d-b99a-4078-af31-bf139da28dea","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:52.844Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"1e0b386d-b99a-4078-af31-bf139da28dea","namespace":"test-ns-chvpt","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T09:50:52.845Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"35438a2f-ee25-4a76-a22b-c9c3467944cf","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:52.845Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"35438a2f-ee25-4a76-a22b-c9c3467944cf","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T09:50:52.846Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"reconcileID":"5261f03a-5526-472b-a10c-98c2463d5a8c"}
{"level":"info","ts":"2026-09-18T09:50:52.870Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"98160d6a-d850-4fbd-8514-e1f2de01192f","rayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:52.870Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"98160d6a-d850-4fbd-8514-e1f2de01192f","pod":"long-running-2qcbf-head-cz5d7","ip":"10.132.0.34"}
{"level":"info","ts":"2026-09-18T09:50:52.870Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"98160d6a-d850-4fbd-8514-e1f2de01192f","pod":"long-running-2qcbf-small-group-worker-94s9h","ip":"10.133.0.32"}
{"level":"info","ts":"2026-09-18T09:50:52.870Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"98160d6a-d850-4fbd-8514-e1f2de01192f","rayCluster":"long-running-2qcbf","podIPs":["10.132.0.34","10.133.0.32"]}
{"level":"info","ts":"2026-09-18T09:50:52.966Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"1e0b386d-b99a-4078-af31-bf139da28dea","name":"kuberay-gateway-access","namespace":"test-ns-chvpt"}
{"level":"info","ts":"2026-09-18T09:50:52.966Z","logger":"controllers.RayJob","msg":"The associated submitter Kubernetes Job for RayJob is deleted","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"35438a2f-ee25-4a76-a22b-c9c3467944cf","Submitter K8s Job":"long-running"}
{"level":"info","ts":"2026-09-18T09:50:52.966Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"35438a2f-ee25-4a76-a22b-c9c3467944cf","isJobDeleted":false}
{"level":"info","ts":"2026-09-18T09:50:52.966Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"35438a2f-ee25-4a76-a22b-c9c3467944cf"}
{"level":"info","ts":"2026-09-18T09:50:52.966Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c66c698f-5ba7-440b-88a9-7254c82498d5","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:52.966Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c66c698f-5ba7-440b-88a9-7254c82498d5","RayCluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:52.966Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c66c698f-5ba7-440b-88a9-7254c82498d5","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T09:50:52.966Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c66c698f-5ba7-440b-88a9-7254c82498d5","Kubernetes Job":""}
{"level":"info","ts":"2026-09-18T09:50:52.966Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c66c698f-5ba7-440b-88a9-7254c82498d5","isJobDeleted":true}
{"level":"info","ts":"2026-09-18T09:50:52.966Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c66c698f-5ba7-440b-88a9-7254c82498d5"}
{"level":"info","ts":"2026-09-18T09:50:52.984Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"1e0b386d-b99a-4078-af31-bf139da28dea","name":"test-ns-chvpt-long-running-2qcbf","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T09:50:52.993Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"1e0b386d-b99a-4078-af31-bf139da28dea","configMap":"kube-rbac-proxy-config-long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:53.002Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"1e0b386d-b99a-4078-af31-bf139da28dea","serviceAccount":"long-running-2qcbf-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:50:53.011Z","logger":"raycluster-resource","msg":"default","name":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:53.011Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-2qcbf","namespace":"test-ns-chvpt"}
{"level":"info","ts":"2026-09-18T09:50:53.066Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"647ec62f-80e8-4d02-b7a3-44897999bf84","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:53.066Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"647ec62f-80e8-4d02-b7a3-44897999bf84","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:53.066Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"647ec62f-80e8-4d02-b7a3-44897999bf84","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T09:50:53.066Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"647ec62f-80e8-4d02-b7a3-44897999bf84","Kubernetes Job":""}
{"level":"info","ts":"2026-09-18T09:50:53.066Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"647ec62f-80e8-4d02-b7a3-44897999bf84","isJobDeleted":true}
{"level":"info","ts":"2026-09-18T09:50:53.066Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"647ec62f-80e8-4d02-b7a3-44897999bf84","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-srdwt","rayClusterName":"long-running-2qcbf","dashboardURL":"long-running-2qcbf-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobDeploymentStatus":"Suspending","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-2qcbf-head-cz5d7","serviceName":"long-running-2qcbf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:49:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:49:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:49:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:50:53.066Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"647ec62f-80e8-4d02-b7a3-44897999bf84","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
{"level":"info","ts":"2026-09-18T09:50:53.066Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"1e0b386d-b99a-4078-af31-bf139da28dea","cluster":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:53.067Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"6c7e63e0-2574-4da3-aded-0ea8f3dc7080"}
{"level":"info","ts":"2026-09-18T09:50:53.067Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"36a62446-3591-4c69-85a6-b9b299f9b6ac","namespacedName":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:53.067Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"86ac7032-4054-4704-8c88-ffbacc7bdc90","namespace":"test-ns-chvpt","clusterName":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:53.067Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"36a62446-3591-4c69-85a6-b9b299f9b6ac"}
{"level":"info","ts":"2026-09-18T09:50:53.067Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"36a62446-3591-4c69-85a6-b9b299f9b6ac","namespace":"test-ns-chvpt","authClusters":0}
{"level":"info","ts":"2026-09-18T09:50:53.067Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"36a62446-3591-4c69-85a6-b9b299f9b6ac","namespace":"test-ns-chvpt"}
{"level":"info","ts":"2026-09-18T09:50:53.067Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"91ac9a6f-d1ee-4659-a63b-39cf600f4ff6"}
{"level":"error","ts":"2026-09-18T09:50:53.075Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"86ac7032-4054-4704-8c88-ffbacc7bdc90","certificate":"ray-head-cert-long-running-2qcbf","error":"certificates.cert-manager.io \"ray-head-cert-long-running-2qcbf\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:53.084Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"797d6462-29dd-400c-854d-4b3a327bc098","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"error","ts":"2026-09-18T09:50:53.094Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"86ac7032-4054-4704-8c88-ffbacc7bdc90","certificate":"ray-worker-cert-long-running-2qcbf","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-2qcbf\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:53.110Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"5d5d8045-4f64-40a8-ac54-3d7331f85096","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:53.125Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"86ac7032-4054-4704-8c88-ffbacc7bdc90","errorCount":2}
{"level":"info","ts":"2026-09-18T09:50:53.125Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"86ac7032-4054-4704-8c88-ffbacc7bdc90"}
{"level":"error","ts":"2026-09-18T09:50:53.125Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"86ac7032-4054-4704-8c88-ffbacc7bdc90","error":"certificates.cert-manager.io \"ray-head-cert-long-running-2qcbf\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:53.130Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"9751649f-a592-45b6-aef0-29a06fbaf9b0","namespace":"test-ns-chvpt","clusterName":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:53.130Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2qcbf","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-2qcbf","reconcileID":"9751649f-a592-45b6-aef0-29a06fbaf9b0","namespace":"test-ns-chvpt","clusterName":"long-running-2qcbf"}
{"level":"info","ts":"2026-09-18T09:50:53.844Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"6ce81263-37d8-48e5-9de3-2275c9d1daf8","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:53.844Z","logger":"controllers.RayJob","msg":"The status is 'Suspended', but the suspend flag is false. Transition the status to 'New'.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"6ce81263-37d8-48e5-9de3-2275c9d1daf8"}
{"level":"info","ts":"2026-09-18T09:50:53.844Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"6ce81263-37d8-48e5-9de3-2275c9d1daf8","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:50:53.844Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"6ce81263-37d8-48e5-9de3-2275c9d1daf8","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspended","new JobDeploymentStatus":""}
{"level":"info","ts":"2026-09-18T09:50:53.859Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"77f26568-1af0-4154-b482-f522ab5fcff0","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:53.859Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"77f26568-1af0-4154-b482-f522ab5fcff0"}
{"level":"info","ts":"2026-09-18T09:50:53.859Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"77f26568-1af0-4154-b482-f522ab5fcff0","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:50:53.859Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"77f26568-1af0-4154-b482-f522ab5fcff0","oldRayJobStatus":{"rayJobInfo":{},"startTime":"2026-09-18T09:49:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:50:53.859Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"77f26568-1af0-4154-b482-f522ab5fcff0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:50:53.875Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0b1040a9-01d4-48d1-8fab-53a6eac41c25","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:53.875Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0b1040a9-01d4-48d1-8fab-53a6eac41c25","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.875Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0b1040a9-01d4-48d1-8fab-53a6eac41c25","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:53.875Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0b1040a9-01d4-48d1-8fab-53a6eac41c25","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:53.880Z","logger":"raycluster-resource","msg":"default","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.880Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-xrs5s","namespace":"test-ns-chvpt"}
{"level":"info","ts":"2026-09-18T09:50:53.892Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0b1040a9-01d4-48d1-8fab-53a6eac41c25","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0b1040a9-01d4-48d1-8fab-53a6eac41c25"}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0b1040a9-01d4-48d1-8fab-53a6eac41c25","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0b1040a9-01d4-48d1-8fab-53a6eac41c25","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"991ddf65-72e2-4366-a3ad-b00aabb6fe77"}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"991ddf65-72e2-4366-a3ad-b00aabb6fe77"}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"991ddf65-72e2-4366-a3ad-b00aabb6fe77"}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"7cba11c8-1fb2-4b7d-8c15-2114c9d77e7b","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"7cba11c8-1fb2-4b7d-8c15-2114c9d77e7b","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"c9d89a57-489d-46a0-95b9-0a772dfac376","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0c4400d1-ae5b-4059-acd8-e0465607d273","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0c4400d1-ae5b-4059-acd8-e0465607d273","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0c4400d1-ae5b-4059-acd8-e0465607d273","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0c4400d1-ae5b-4059-acd8-e0465607d273"}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0c4400d1-ae5b-4059-acd8-e0465607d273","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"0c4400d1-ae5b-4059-acd8-e0465607d273","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"f35a3531-6dfd-481d-a887-e4e414e1fec6","namespacedName":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"f35a3531-6dfd-481d-a887-e4e414e1fec6","mode":"IntegratedOAuth","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"f35a3531-6dfd-481d-a887-e4e414e1fec6","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.893Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"f35a3531-6dfd-481d-a887-e4e414e1fec6","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.900Z","logger":"raycluster-resource","msg":"default","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.900Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-xrs5s","namespace":"test-ns-chvpt"}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"991ddf65-72e2-4366-a3ad-b00aabb6fe77","name":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"991ddf65-72e2-4366-a3ad-b00aabb6fe77","error":"head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"error","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"991ddf65-72e2-4366-a3ad-b00aabb6fe77","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"991ddf65-72e2-4366-a3ad-b00aabb6fe77","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"991ddf65-72e2-4366-a3ad-b00aabb6fe77","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"991ddf65-72e2-4366-a3ad-b00aabb6fe77","serviceSelectors":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/identifier":"long-running-xrs5s-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"991ddf65-72e2-4366-a3ad-b00aabb6fe77","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"991ddf65-72e2-4366-a3ad-b00aabb6fe77","error":"unable to find head service. cluster name long-running-xrs5s, filter labels [test-ns-chvpt map[ray.io/cluster:long-running-xrs5s ray.io/identifier:long-running-xrs5s-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"991ddf65-72e2-4366-a3ad-b00aabb6fe77"}
{"level":"error","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"991ddf65-72e2-4366-a3ad-b00aabb6fe77","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"6a9eba23-4979-45dc-a522-2fc9f715070a"}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"6a9eba23-4979-45dc-a522-2fc9f715070a"}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"6a9eba23-4979-45dc-a522-2fc9f715070a"}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"6a9eba23-4979-45dc-a522-2fc9f715070a","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"6a9eba23-4979-45dc-a522-2fc9f715070a","error":"head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"error","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"6a9eba23-4979-45dc-a522-2fc9f715070a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"6a9eba23-4979-45dc-a522-2fc9f715070a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"6a9eba23-4979-45dc-a522-2fc9f715070a","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"6a9eba23-4979-45dc-a522-2fc9f715070a","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"6a9eba23-4979-45dc-a522-2fc9f715070a","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:53.903Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"6a9eba23-4979-45dc-a522-2fc9f715070a","name":"long-running-xrs5s","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:50:53.910Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"7cba11c8-1fb2-4b7d-8c15-2114c9d77e7b","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.911Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"7cba11c8-1fb2-4b7d-8c15-2114c9d77e7b","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.913Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"20c83a8b-e17c-4b5f-ba79-79a658d4f67f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:53.913Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"20c83a8b-e17c-4b5f-ba79-79a658d4f67f","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.913Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"20c83a8b-e17c-4b5f-ba79-79a658d4f67f","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:53.914Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"20c83a8b-e17c-4b5f-ba79-79a658d4f67f"}
{"level":"info","ts":"2026-09-18T09:50:53.914Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"20c83a8b-e17c-4b5f-ba79-79a658d4f67f","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:50:53.914Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"20c83a8b-e17c-4b5f-ba79-79a658d4f67f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:50:53.966Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"c9d89a57-489d-46a0-95b9-0a772dfac376","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.966Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"6a9eba23-4979-45dc-a522-2fc9f715070a","name":"long-running-xrs5s","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-xrs5s\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-chvpt","name":"long-running-xrs5s"}}
{"level":"info","ts":"2026-09-18T09:50:53.966Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"6a9eba23-4979-45dc-a522-2fc9f715070a"}
{"level":"error","ts":"2026-09-18T09:50:53.966Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"6a9eba23-4979-45dc-a522-2fc9f715070a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:53.966Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"8fc37480-3b9f-4d3e-bc0a-e5e73ae13894","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.966Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"7d41a98a-2040-4ba5-97d1-7e62f92b7915"}
{"level":"info","ts":"2026-09-18T09:50:53.966Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"7d41a98a-2040-4ba5-97d1-7e62f92b7915"}
{"level":"info","ts":"2026-09-18T09:50:53.966Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"7d41a98a-2040-4ba5-97d1-7e62f92b7915"}
{"level":"info","ts":"2026-09-18T09:50:53.966Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"7d41a98a-2040-4ba5-97d1-7e62f92b7915","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:53.966Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"7d41a98a-2040-4ba5-97d1-7e62f92b7915","error":"head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"error","ts":"2026-09-18T09:50:53.966Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"7d41a98a-2040-4ba5-97d1-7e62f92b7915","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:53.966Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"7d41a98a-2040-4ba5-97d1-7e62f92b7915","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"info","ts":"2026-09-18T09:50:53.966Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"7d41a98a-2040-4ba5-97d1-7e62f92b7915","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:53.966Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"7d41a98a-2040-4ba5-97d1-7e62f92b7915","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:53.966Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"7d41a98a-2040-4ba5-97d1-7e62f92b7915","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:53.966Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"7d41a98a-2040-4ba5-97d1-7e62f92b7915","name":"long-running-xrs5s","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:50:53.966Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"7cba11c8-1fb2-4b7d-8c15-2114c9d77e7b","rayCluster":"long-running-xrs5s","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T09:50:53.966Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"7cba11c8-1fb2-4b7d-8c15-2114c9d77e7b","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.967Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"f35a3531-6dfd-481d-a887-e4e414e1fec6","name":"long-running-xrs5s-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:50:53.975Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"f35a3531-6dfd-481d-a887-e4e414e1fec6","name":"kuberay-gateway-access","namespace":"test-ns-chvpt","operation":"created"}
{"level":"info","ts":"2026-09-18T09:50:53.980Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"8fc37480-3b9f-4d3e-bc0a-e5e73ae13894","name":"long-running-xrs5s-head"}
{"level":"info","ts":"2026-09-18T09:50:53.981Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"7cba11c8-1fb2-4b7d-8c15-2114c9d77e7b","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.987Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"f35a3531-6dfd-481d-a887-e4e414e1fec6","name":"test-ns-chvpt-long-running-xrs5s","operation":"created"}
{"level":"info","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72695fee-c153-47c2-830d-30e48a2a3ced","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72695fee-c153-47c2-830d-30e48a2a3ced","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"7d41a98a-2040-4ba5-97d1-7e62f92b7915"}
{"level":"info","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72695fee-c153-47c2-830d-30e48a2a3ced","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"error","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"7d41a98a-2040-4ba5-97d1-7e62f92b7915","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"a52453f5-ad5a-48f4-a559-94dd0e64f89a"}
{"level":"info","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"a52453f5-ad5a-48f4-a559-94dd0e64f89a"}
{"level":"info","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"a52453f5-ad5a-48f4-a559-94dd0e64f89a"}
{"level":"info","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"a52453f5-ad5a-48f4-a559-94dd0e64f89a","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72695fee-c153-47c2-830d-30e48a2a3ced"}
{"level":"info","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"a52453f5-ad5a-48f4-a559-94dd0e64f89a","error":"head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"error","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"a52453f5-ad5a-48f4-a559-94dd0e64f89a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"a52453f5-ad5a-48f4-a559-94dd0e64f89a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"info","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72695fee-c153-47c2-830d-30e48a2a3ced","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72695fee-c153-47c2-830d-30e48a2a3ced","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"72695fee-c153-47c2-830d-30e48a2a3ced","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:50:53.995Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"f35a3531-6dfd-481d-a887-e4e414e1fec6","name":"kube-rbac-proxy-config-long-running-xrs5s","operation":"created"}
{"level":"info","ts":"2026-09-18T09:50:53.987Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"a52453f5-ad5a-48f4-a559-94dd0e64f89a","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"a52453f5-ad5a-48f4-a559-94dd0e64f89a","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"a52453f5-ad5a-48f4-a559-94dd0e64f89a","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"a52453f5-ad5a-48f4-a559-94dd0e64f89a"}
{"level":"error","ts":"2026-09-18T09:50:54.066Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"a52453f5-ad5a-48f4-a559-94dd0e64f89a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"03730ce7-1c14-479a-8a05-54d4cd028a62"}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"03730ce7-1c14-479a-8a05-54d4cd028a62"}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"03730ce7-1c14-479a-8a05-54d4cd028a62"}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"03730ce7-1c14-479a-8a05-54d4cd028a62","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"7cba11c8-1fb2-4b7d-8c15-2114c9d77e7b","rayCluster":"long-running-xrs5s","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"03730ce7-1c14-479a-8a05-54d4cd028a62","error":"head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"error","ts":"2026-09-18T09:50:54.066Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"03730ce7-1c14-479a-8a05-54d4cd028a62","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"03730ce7-1c14-479a-8a05-54d4cd028a62","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"03730ce7-1c14-479a-8a05-54d4cd028a62","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"03730ce7-1c14-479a-8a05-54d4cd028a62","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"03730ce7-1c14-479a-8a05-54d4cd028a62","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"03730ce7-1c14-479a-8a05-54d4cd028a62"}
{"level":"error","ts":"2026-09-18T09:50:54.066Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"03730ce7-1c14-479a-8a05-54d4cd028a62","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"8fc37480-3b9f-4d3e-bc0a-e5e73ae13894","name":"long-running-xrs5s-workers"}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"8fc37480-3b9f-4d3e-bc0a-e5e73ae13894","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"b377713f-a131-415a-9fc8-b9ed878e546e","cluster":"long-running-xrs5s"}
{"level":"error","ts":"2026-09-18T09:50:54.066Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"f35a3531-6dfd-481d-a887-e4e414e1fec6","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-xrs5s\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"f35a3531-6dfd-481d-a887-e4e414e1fec6"}
{"level":"error","ts":"2026-09-18T09:50:54.066Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"f35a3531-6dfd-481d-a887-e4e414e1fec6","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-xrs5s\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"5f8d6d8a-9c09-4221-a675-59057fa99cb6","namespacedName":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"5f8d6d8a-9c09-4221-a675-59057fa99cb6","mode":"IntegratedOAuth","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.066Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"5f8d6d8a-9c09-4221-a675-59057fa99cb6","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.067Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"689b7aad-4c91-41e3-b2cf-42783f379ffe","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:54.067Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"689b7aad-4c91-41e3-b2cf-42783f379ffe","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.067Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"689b7aad-4c91-41e3-b2cf-42783f379ffe","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:54.067Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"689b7aad-4c91-41e3-b2cf-42783f379ffe"}
{"level":"info","ts":"2026-09-18T09:50:54.067Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"689b7aad-4c91-41e3-b2cf-42783f379ffe","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:50:54.067Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"689b7aad-4c91-41e3-b2cf-42783f379ffe","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:54.085Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"b377713f-a131-415a-9fc8-b9ed878e546e","name":"long-running-xrs5s-head"}
{"level":"info","ts":"2026-09-18T09:50:54.086Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"7cba11c8-1fb2-4b7d-8c15-2114c9d77e7b","rayCluster":"long-running-xrs5s","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:50:54.086Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"7cba11c8-1fb2-4b7d-8c15-2114c9d77e7b"}
{"level":"info","ts":"2026-09-18T09:50:54.086Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"93d95698-e062-4cc2-b995-03bd8ba4eedb","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.087Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"93d95698-e062-4cc2-b995-03bd8ba4eedb"}
{"level":"info","ts":"2026-09-18T09:50:54.087Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"5f8d6d8a-9c09-4221-a675-59057fa99cb6","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.087Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"9ea728ed-7121-44a6-9122-d531e3a56292","namespacedName":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:54.087Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"9ea728ed-7121-44a6-9122-d531e3a56292","mode":"IntegratedOAuth","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.087Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"9ea728ed-7121-44a6-9122-d531e3a56292","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.088Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"3a34273e-0d90-4133-867b-8702a85e81fa","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.088Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"3a34273e-0d90-4133-867b-8702a85e81fa"}
{"level":"info","ts":"2026-09-18T09:50:54.088Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb1e131f-19bc-4c86-99dc-03788a52929a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:54.088Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb1e131f-19bc-4c86-99dc-03788a52929a","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.088Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb1e131f-19bc-4c86-99dc-03788a52929a","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:54.088Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb1e131f-19bc-4c86-99dc-03788a52929a"}
{"level":"info","ts":"2026-09-18T09:50:54.088Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb1e131f-19bc-4c86-99dc-03788a52929a","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:50:54.088Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb1e131f-19bc-4c86-99dc-03788a52929a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:54.088Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"bb1e131f-19bc-4c86-99dc-03788a52929a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"error","ts":"2026-09-18T09:50:54.098Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"9ea728ed-7121-44a6-9122-d531e3a56292","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-xrs5s\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:54.098Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"9ea728ed-7121-44a6-9122-d531e3a56292"}
{"level":"error","ts":"2026-09-18T09:50:54.098Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"9ea728ed-7121-44a6-9122-d531e3a56292","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-xrs5s\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:54.103Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"b377713f-a131-415a-9fc8-b9ed878e546e","name":"long-running-xrs5s-workers"}
{"level":"info","ts":"2026-09-18T09:50:54.103Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"b377713f-a131-415a-9fc8-b9ed878e546e","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.103Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"2087ede1-f3f5-4b7b-a8ec-620f6c3f6145","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.103Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"06af4a99-8e29-42aa-afe0-fad882ca0b97","namespacedName":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:54.103Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"06af4a99-8e29-42aa-afe0-fad882ca0b97","mode":"IntegratedOAuth","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.103Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"06af4a99-8e29-42aa-afe0-fad882ca0b97","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.106Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"034333d5-faa4-462e-a631-429f0983b290"}
{"level":"info","ts":"2026-09-18T09:50:54.106Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"034333d5-faa4-462e-a631-429f0983b290"}
{"level":"info","ts":"2026-09-18T09:50:54.106Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"034333d5-faa4-462e-a631-429f0983b290"}
{"level":"info","ts":"2026-09-18T09:50:54.106Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"034333d5-faa4-462e-a631-429f0983b290","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:54.106Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"034333d5-faa4-462e-a631-429f0983b290","error":"head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"error","ts":"2026-09-18T09:50:54.106Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"034333d5-faa4-462e-a631-429f0983b290","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:54.106Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"034333d5-faa4-462e-a631-429f0983b290","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"info","ts":"2026-09-18T09:50:54.106Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"034333d5-faa4-462e-a631-429f0983b290","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:54.106Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"034333d5-faa4-462e-a631-429f0983b290","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:54.106Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"034333d5-faa4-462e-a631-429f0983b290","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:54.106Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"034333d5-faa4-462e-a631-429f0983b290"}
{"level":"error","ts":"2026-09-18T09:50:54.106Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"034333d5-faa4-462e-a631-429f0983b290","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:54.166Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"06af4a99-8e29-42aa-afe0-fad882ca0b97","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.167Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c76a95aa-09bb-4c5b-9cba-98c56d8b0ca5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:54.167Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c76a95aa-09bb-4c5b-9cba-98c56d8b0ca5","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.167Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c76a95aa-09bb-4c5b-9cba-98c56d8b0ca5","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:54.167Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c76a95aa-09bb-4c5b-9cba-98c56d8b0ca5"}
{"level":"info","ts":"2026-09-18T09:50:54.167Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c76a95aa-09bb-4c5b-9cba-98c56d8b0ca5","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:50:54.167Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c76a95aa-09bb-4c5b-9cba-98c56d8b0ca5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:54.175Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"2087ede1-f3f5-4b7b-a8ec-620f6c3f6145","name":"long-running-xrs5s-head"}
{"level":"info","ts":"2026-09-18T09:50:54.193Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"2087ede1-f3f5-4b7b-a8ec-620f6c3f6145","name":"long-running-xrs5s-workers"}
{"level":"info","ts":"2026-09-18T09:50:54.193Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"2087ede1-f3f5-4b7b-a8ec-620f6c3f6145","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:54.266Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"826c95e0-d215-4c0e-ae93-a26b0414c2b8"}
{"level":"info","ts":"2026-09-18T09:50:54.266Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"826c95e0-d215-4c0e-ae93-a26b0414c2b8"}
{"level":"info","ts":"2026-09-18T09:50:54.266Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"826c95e0-d215-4c0e-ae93-a26b0414c2b8"}
{"level":"info","ts":"2026-09-18T09:50:54.267Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"826c95e0-d215-4c0e-ae93-a26b0414c2b8","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:54.267Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"826c95e0-d215-4c0e-ae93-a26b0414c2b8","error":"head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"error","ts":"2026-09-18T09:50:54.267Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"826c95e0-d215-4c0e-ae93-a26b0414c2b8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:54.267Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"826c95e0-d215-4c0e-ae93-a26b0414c2b8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"info","ts":"2026-09-18T09:50:54.267Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"826c95e0-d215-4c0e-ae93-a26b0414c2b8","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:54.267Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"826c95e0-d215-4c0e-ae93-a26b0414c2b8","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:54.267Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"826c95e0-d215-4c0e-ae93-a26b0414c2b8","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:54.267Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"826c95e0-d215-4c0e-ae93-a26b0414c2b8"}
{"level":"error","ts":"2026-09-18T09:50:54.267Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"826c95e0-d215-4c0e-ae93-a26b0414c2b8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:54.587Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"65711126-df58-44b9-a36a-19ed06b3e2da"}
{"level":"info","ts":"2026-09-18T09:50:54.587Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"65711126-df58-44b9-a36a-19ed06b3e2da"}
{"level":"info","ts":"2026-09-18T09:50:54.587Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"65711126-df58-44b9-a36a-19ed06b3e2da"}
{"level":"info","ts":"2026-09-18T09:50:54.587Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"65711126-df58-44b9-a36a-19ed06b3e2da","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:54.588Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"65711126-df58-44b9-a36a-19ed06b3e2da","error":"head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"error","ts":"2026-09-18T09:50:54.588Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"65711126-df58-44b9-a36a-19ed06b3e2da","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:54.588Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"65711126-df58-44b9-a36a-19ed06b3e2da","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"info","ts":"2026-09-18T09:50:54.588Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"65711126-df58-44b9-a36a-19ed06b3e2da","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:54.588Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"65711126-df58-44b9-a36a-19ed06b3e2da","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:54.588Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"65711126-df58-44b9-a36a-19ed06b3e2da","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:54.588Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"65711126-df58-44b9-a36a-19ed06b3e2da"}
{"level":"error","ts":"2026-09-18T09:50:54.588Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"65711126-df58-44b9-a36a-19ed06b3e2da","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:55.228Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"3d4096db-8045-40bd-9c03-790caf74b479"}
{"level":"info","ts":"2026-09-18T09:50:55.228Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"3d4096db-8045-40bd-9c03-790caf74b479"}
{"level":"info","ts":"2026-09-18T09:50:55.228Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"3d4096db-8045-40bd-9c03-790caf74b479"}
{"level":"info","ts":"2026-09-18T09:50:55.228Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"3d4096db-8045-40bd-9c03-790caf74b479","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:55.228Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"3d4096db-8045-40bd-9c03-790caf74b479","error":"head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"error","ts":"2026-09-18T09:50:55.228Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"3d4096db-8045-40bd-9c03-790caf74b479","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:55.228Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"3d4096db-8045-40bd-9c03-790caf74b479","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found"}
{"level":"info","ts":"2026-09-18T09:50:55.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"3d4096db-8045-40bd-9c03-790caf74b479","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:55.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"3d4096db-8045-40bd-9c03-790caf74b479","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:55.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"3d4096db-8045-40bd-9c03-790caf74b479","filter labels":["test-ns-chvpt",{"ray.io/cluster":"long-running-xrs5s","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:50:55.228Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"3d4096db-8045-40bd-9c03-790caf74b479"}
{"level":"error","ts":"2026-09-18T09:50:55.228Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"3d4096db-8045-40bd-9c03-790caf74b479","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-xrs5s not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:50:56.509Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204"}
{"level":"info","ts":"2026-09-18T09:50:56.509Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204"}
{"level":"info","ts":"2026-09-18T09:50:56.509Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204"}
{"level":"info","ts":"2026-09-18T09:50:56.509Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:56.509Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204"}
{"level":"info","ts":"2026-09-18T09:50:56.509Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:50:56.509Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204"}
{"level":"info","ts":"2026-09-18T09:50:56.509Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","mode":"IntegratedOAuth","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:56.509Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","cluster":"long-running-xrs5s","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:50:56.509Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","cluster":"long-running-xrs5s","authType":"IntegratedOAuth","serviceAccount":"long-running-xrs5s-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:50:56.509Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-xrs5s","ray.io/group":"headgroup","ray.io/identifier":"long-running-xrs5s-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:50:56.509Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:50:56.509Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:50:56.510Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:50:56.510Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","enabled":true}
{"level":"info","ts":"2026-09-18T09:50:56.528Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","name":"long-running-xrs5s-head-lcfg7"}
{"level":"info","ts":"2026-09-18T09:50:56.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:50:56.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:56.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:50:56.528Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:50:56.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:56.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:50:56.528Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204"}
{"level":"info","ts":"2026-09-18T09:50:56.528Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","nodeType":"worker","rayStartParams":{"address":"long-running-xrs5s-head-svc.test-ns-chvpt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:50:56.528Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","rayStartCmd":"ray start  --address=long-running-xrs5s-head-svc.test-ns-chvpt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:50:56.528Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-xrs5s-head-svc.test-ns-chvpt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:50:56.528Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","enabled":true}
{"level":"info","ts":"2026-09-18T09:50:56.548Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","name":"long-running-xrs5s-small-group-worker-cqdn2"}
{"level":"info","ts":"2026-09-18T09:50:56.548Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:56.549Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:56.549Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"44be221b-69ec-4932-adf6-b2008d75a204","name":"long-running-xrs5s","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"68b199f0-0b38-4f07-afc5-2ecf657d680f"}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"68b199f0-0b38-4f07-afc5-2ecf657d680f"}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"68b199f0-0b38-4f07-afc5-2ecf657d680f"}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"68b199f0-0b38-4f07-afc5-2ecf657d680f","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c98d248e-5f3a-4113-a978-28210aca4989","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c98d248e-5f3a-4113-a978-28210aca4989","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c98d248e-5f3a-4113-a978-28210aca4989","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"68b199f0-0b38-4f07-afc5-2ecf657d680f","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"68b199f0-0b38-4f07-afc5-2ecf657d680f","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"68b199f0-0b38-4f07-afc5-2ecf657d680f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"68b199f0-0b38-4f07-afc5-2ecf657d680f","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"68b199f0-0b38-4f07-afc5-2ecf657d680f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"68b199f0-0b38-4f07-afc5-2ecf657d680f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"68b199f0-0b38-4f07-afc5-2ecf657d680f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"68b199f0-0b38-4f07-afc5-2ecf657d680f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"01232dff-7804-4100-a34c-dad7c2818096","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"68b199f0-0b38-4f07-afc5-2ecf657d680f","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"68b199f0-0b38-4f07-afc5-2ecf657d680f","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"01232dff-7804-4100-a34c-dad7c2818096","rayCluster":"long-running-xrs5s","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"68b199f0-0b38-4f07-afc5-2ecf657d680f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"68b199f0-0b38-4f07-afc5-2ecf657d680f","seconds":300}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c98d248e-5f3a-4113-a978-28210aca4989"}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c98d248e-5f3a-4113-a978-28210aca4989","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"2ce857ea-955f-4b57-bb71-1044a0fe0027","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c98d248e-5f3a-4113-a978-28210aca4989","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:56.571Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c98d248e-5f3a-4113-a978-28210aca4989","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"33f26375-e457-4162-b626-e4e044bdecec"}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"33f26375-e457-4162-b626-e4e044bdecec"}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d2849792-604c-4e54-bfba-2c38f5c0b7ac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d2849792-604c-4e54-bfba-2c38f5c0b7ac","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"33f26375-e457-4162-b626-e4e044bdecec"}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d2849792-604c-4e54-bfba-2c38f5c0b7ac","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"33f26375-e457-4162-b626-e4e044bdecec","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"33f26375-e457-4162-b626-e4e044bdecec","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"33f26375-e457-4162-b626-e4e044bdecec","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"33f26375-e457-4162-b626-e4e044bdecec","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d2849792-604c-4e54-bfba-2c38f5c0b7ac"}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d2849792-604c-4e54-bfba-2c38f5c0b7ac","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"33f26375-e457-4162-b626-e4e044bdecec","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"33f26375-e457-4162-b626-e4e044bdecec","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"33f26375-e457-4162-b626-e4e044bdecec","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"33f26375-e457-4162-b626-e4e044bdecec","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d2849792-604c-4e54-bfba-2c38f5c0b7ac","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"33f26375-e457-4162-b626-e4e044bdecec","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"33f26375-e457-4162-b626-e4e044bdecec","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"33f26375-e457-4162-b626-e4e044bdecec","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"33f26375-e457-4162-b626-e4e044bdecec","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:50:56.595Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"33f26375-e457-4162-b626-e4e044bdecec","seconds":300}
{"level":"info","ts":"2026-09-18T09:50:56.596Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"2ce857ea-955f-4b57-bb71-1044a0fe0027","name":"long-running-xrs5s-head"}
{"level":"info","ts":"2026-09-18T09:50:56.598Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"21ac24d0-69e4-4fd0-8349-64829b54407f"}
{"level":"info","ts":"2026-09-18T09:50:56.598Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"21ac24d0-69e4-4fd0-8349-64829b54407f"}
{"level":"info","ts":"2026-09-18T09:50:56.598Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"21ac24d0-69e4-4fd0-8349-64829b54407f"}
{"level":"info","ts":"2026-09-18T09:50:56.598Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"21ac24d0-69e4-4fd0-8349-64829b54407f","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:56.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"21ac24d0-69e4-4fd0-8349-64829b54407f","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:50:56.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"21ac24d0-69e4-4fd0-8349-64829b54407f","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:56.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"21ac24d0-69e4-4fd0-8349-64829b54407f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:50:56.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"21ac24d0-69e4-4fd0-8349-64829b54407f","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:56.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"21ac24d0-69e4-4fd0-8349-64829b54407f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:56.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"21ac24d0-69e4-4fd0-8349-64829b54407f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:50:56.598Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"21ac24d0-69e4-4fd0-8349-64829b54407f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:50:56.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"21ac24d0-69e4-4fd0-8349-64829b54407f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:56.598Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"21ac24d0-69e4-4fd0-8349-64829b54407f","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:56.598Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"21ac24d0-69e4-4fd0-8349-64829b54407f","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:56.598Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"21ac24d0-69e4-4fd0-8349-64829b54407f","name":"long-running-xrs5s","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:50:56.614Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"2ce857ea-955f-4b57-bb71-1044a0fe0027","name":"long-running-xrs5s-workers"}
{"level":"info","ts":"2026-09-18T09:50:56.614Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"2ce857ea-955f-4b57-bb71-1044a0fe0027","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:56.616Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"7a65bd19-ed5b-46f7-bd57-11d8e08a3c78","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:56.616Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b52e7343-c317-4941-86c0-d34cba7abca6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:56.616Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b52e7343-c317-4941-86c0-d34cba7abca6","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:56.616Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b52e7343-c317-4941-86c0-d34cba7abca6","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:56.617Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"0ec36d15-9b60-4e09-b098-5ed7c47be8d3","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:56.617Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b52e7343-c317-4941-86c0-d34cba7abca6"}
{"level":"info","ts":"2026-09-18T09:50:56.617Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b52e7343-c317-4941-86c0-d34cba7abca6","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:50:56.617Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"0ec36d15-9b60-4e09-b098-5ed7c47be8d3","rayCluster":"long-running-xrs5s","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:50:56.617Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b52e7343-c317-4941-86c0-d34cba7abca6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:56.617Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b52e7343-c317-4941-86c0-d34cba7abca6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:50:56.666Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"c46c8d26-341b-444c-91e3-e866f29d7db7"}
{"level":"info","ts":"2026-09-18T09:50:56.666Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"c46c8d26-341b-444c-91e3-e866f29d7db7"}
{"level":"info","ts":"2026-09-18T09:50:56.666Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"c46c8d26-341b-444c-91e3-e866f29d7db7"}
{"level":"info","ts":"2026-09-18T09:50:56.666Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"c46c8d26-341b-444c-91e3-e866f29d7db7","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:56.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"c46c8d26-341b-444c-91e3-e866f29d7db7","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:50:56.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"c46c8d26-341b-444c-91e3-e866f29d7db7","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:56.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"c46c8d26-341b-444c-91e3-e866f29d7db7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:50:56.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"c46c8d26-341b-444c-91e3-e866f29d7db7","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:56.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"c46c8d26-341b-444c-91e3-e866f29d7db7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:56.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"c46c8d26-341b-444c-91e3-e866f29d7db7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:50:56.666Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"c46c8d26-341b-444c-91e3-e866f29d7db7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:50:56.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"c46c8d26-341b-444c-91e3-e866f29d7db7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:56.667Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"c46c8d26-341b-444c-91e3-e866f29d7db7","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:56.667Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"c46c8d26-341b-444c-91e3-e866f29d7db7","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:56.667Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"c46c8d26-341b-444c-91e3-e866f29d7db7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:50:56.667Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"c46c8d26-341b-444c-91e3-e866f29d7db7","seconds":300}
{"level":"info","ts":"2026-09-18T09:50:56.673Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"7a65bd19-ed5b-46f7-bd57-11d8e08a3c78","name":"long-running-xrs5s-head"}
{"level":"info","ts":"2026-09-18T09:50:56.684Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"747c9741-9af9-4b10-98ef-cdf0a50be239","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:56.684Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"747c9741-9af9-4b10-98ef-cdf0a50be239","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:56.684Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"747c9741-9af9-4b10-98ef-cdf0a50be239","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:56.684Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"747c9741-9af9-4b10-98ef-cdf0a50be239"}
{"level":"info","ts":"2026-09-18T09:50:56.684Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"747c9741-9af9-4b10-98ef-cdf0a50be239","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:50:56.684Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"747c9741-9af9-4b10-98ef-cdf0a50be239","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:56.704Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"7a65bd19-ed5b-46f7-bd57-11d8e08a3c78","name":"long-running-xrs5s-workers"}
{"level":"info","ts":"2026-09-18T09:50:56.704Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"7a65bd19-ed5b-46f7-bd57-11d8e08a3c78","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:56.859Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"81879bf3-46f0-4586-b84e-b020527bbcb1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:56.859Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"81879bf3-46f0-4586-b84e-b020527bbcb1","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:56.859Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"81879bf3-46f0-4586-b84e-b020527bbcb1","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:56.859Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"81879bf3-46f0-4586-b84e-b020527bbcb1"}
{"level":"info","ts":"2026-09-18T09:50:56.859Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"81879bf3-46f0-4586-b84e-b020527bbcb1","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:50:56.859Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"81879bf3-46f0-4586-b84e-b020527bbcb1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:57.146Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"460e6feb-f0b7-4e46-aad9-a6440ad625d6"}
{"level":"info","ts":"2026-09-18T09:50:57.146Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"460e6feb-f0b7-4e46-aad9-a6440ad625d6"}
{"level":"info","ts":"2026-09-18T09:50:57.146Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"460e6feb-f0b7-4e46-aad9-a6440ad625d6"}
{"level":"info","ts":"2026-09-18T09:50:57.146Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"460e6feb-f0b7-4e46-aad9-a6440ad625d6","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:57.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"460e6feb-f0b7-4e46-aad9-a6440ad625d6","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:50:57.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"460e6feb-f0b7-4e46-aad9-a6440ad625d6","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:57.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"460e6feb-f0b7-4e46-aad9-a6440ad625d6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:50:57.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"460e6feb-f0b7-4e46-aad9-a6440ad625d6","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:57.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"460e6feb-f0b7-4e46-aad9-a6440ad625d6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:57.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"460e6feb-f0b7-4e46-aad9-a6440ad625d6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:50:57.147Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"460e6feb-f0b7-4e46-aad9-a6440ad625d6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:50:57.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"460e6feb-f0b7-4e46-aad9-a6440ad625d6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:57.147Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"460e6feb-f0b7-4e46-aad9-a6440ad625d6","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:57.147Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"460e6feb-f0b7-4e46-aad9-a6440ad625d6","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:57.147Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"460e6feb-f0b7-4e46-aad9-a6440ad625d6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:50:57.147Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"460e6feb-f0b7-4e46-aad9-a6440ad625d6","seconds":300}
{"level":"info","ts":"2026-09-18T09:50:57.167Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4ef7135e-e712-43bf-8a45-733c3cb83e1d"}
{"level":"info","ts":"2026-09-18T09:50:57.167Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4ef7135e-e712-43bf-8a45-733c3cb83e1d"}
{"level":"info","ts":"2026-09-18T09:50:57.167Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4ef7135e-e712-43bf-8a45-733c3cb83e1d"}
{"level":"info","ts":"2026-09-18T09:50:57.167Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4ef7135e-e712-43bf-8a45-733c3cb83e1d","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:57.168Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4ef7135e-e712-43bf-8a45-733c3cb83e1d","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:50:57.168Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4ef7135e-e712-43bf-8a45-733c3cb83e1d","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:57.168Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4ef7135e-e712-43bf-8a45-733c3cb83e1d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:50:57.168Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4ef7135e-e712-43bf-8a45-733c3cb83e1d","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:57.168Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4ef7135e-e712-43bf-8a45-733c3cb83e1d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:57.168Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4ef7135e-e712-43bf-8a45-733c3cb83e1d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:50:57.168Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4ef7135e-e712-43bf-8a45-733c3cb83e1d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:50:57.168Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4ef7135e-e712-43bf-8a45-733c3cb83e1d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:57.168Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4ef7135e-e712-43bf-8a45-733c3cb83e1d","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:57.168Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4ef7135e-e712-43bf-8a45-733c3cb83e1d","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:57.168Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4ef7135e-e712-43bf-8a45-733c3cb83e1d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:50:57.168Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4ef7135e-e712-43bf-8a45-733c3cb83e1d","seconds":300}
{"level":"info","ts":"2026-09-18T09:50:57.464Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8d4ab1b6-64d6-4ea8-aef2-ee53d483dd87"}
{"level":"info","ts":"2026-09-18T09:50:57.464Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8d4ab1b6-64d6-4ea8-aef2-ee53d483dd87"}
{"level":"info","ts":"2026-09-18T09:50:57.464Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8d4ab1b6-64d6-4ea8-aef2-ee53d483dd87"}
{"level":"info","ts":"2026-09-18T09:50:57.464Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8d4ab1b6-64d6-4ea8-aef2-ee53d483dd87","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:57.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8d4ab1b6-64d6-4ea8-aef2-ee53d483dd87","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:50:57.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8d4ab1b6-64d6-4ea8-aef2-ee53d483dd87","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:57.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8d4ab1b6-64d6-4ea8-aef2-ee53d483dd87","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:50:57.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8d4ab1b6-64d6-4ea8-aef2-ee53d483dd87","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:57.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8d4ab1b6-64d6-4ea8-aef2-ee53d483dd87","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:57.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8d4ab1b6-64d6-4ea8-aef2-ee53d483dd87","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:50:57.464Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8d4ab1b6-64d6-4ea8-aef2-ee53d483dd87","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:50:57.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8d4ab1b6-64d6-4ea8-aef2-ee53d483dd87","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:57.464Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8d4ab1b6-64d6-4ea8-aef2-ee53d483dd87","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:57.464Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8d4ab1b6-64d6-4ea8-aef2-ee53d483dd87","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:57.465Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8d4ab1b6-64d6-4ea8-aef2-ee53d483dd87","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:50:57.465Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8d4ab1b6-64d6-4ea8-aef2-ee53d483dd87","seconds":300}
{"level":"info","ts":"2026-09-18T09:50:57.726Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"58dcbb0b-ff30-4ebf-b54f-468bdfc60b6d"}
{"level":"info","ts":"2026-09-18T09:50:57.726Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"58dcbb0b-ff30-4ebf-b54f-468bdfc60b6d"}
{"level":"info","ts":"2026-09-18T09:50:57.726Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"58dcbb0b-ff30-4ebf-b54f-468bdfc60b6d"}
{"level":"info","ts":"2026-09-18T09:50:57.726Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"58dcbb0b-ff30-4ebf-b54f-468bdfc60b6d","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:57.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"58dcbb0b-ff30-4ebf-b54f-468bdfc60b6d","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:50:57.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"58dcbb0b-ff30-4ebf-b54f-468bdfc60b6d","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:57.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"58dcbb0b-ff30-4ebf-b54f-468bdfc60b6d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:50:57.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"58dcbb0b-ff30-4ebf-b54f-468bdfc60b6d","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:57.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"58dcbb0b-ff30-4ebf-b54f-468bdfc60b6d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:57.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"58dcbb0b-ff30-4ebf-b54f-468bdfc60b6d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:50:57.726Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"58dcbb0b-ff30-4ebf-b54f-468bdfc60b6d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:50:57.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"58dcbb0b-ff30-4ebf-b54f-468bdfc60b6d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:57.726Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"58dcbb0b-ff30-4ebf-b54f-468bdfc60b6d","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:57.727Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"58dcbb0b-ff30-4ebf-b54f-468bdfc60b6d","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:57.727Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"58dcbb0b-ff30-4ebf-b54f-468bdfc60b6d","name":"long-running-xrs5s","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:50:57.749Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"26c4d33a-ef1c-4256-a5fb-7328c52c4d31","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:57.749Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"00a35ec5-ea12-4505-8870-e3645f5dbf0c","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:57.749Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"511aabef-0612-4d81-b3ef-60afd83da78c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:57.749Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"511aabef-0612-4d81-b3ef-60afd83da78c","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:57.749Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"00a35ec5-ea12-4505-8870-e3645f5dbf0c","pod":"long-running-xrs5s-head-lcfg7","ip":"10.132.0.35"}
{"level":"info","ts":"2026-09-18T09:50:57.749Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"00a35ec5-ea12-4505-8870-e3645f5dbf0c","pod":"long-running-xrs5s-small-group-worker-cqdn2","ip":"10.133.0.34"}
{"level":"info","ts":"2026-09-18T09:50:57.749Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"511aabef-0612-4d81-b3ef-60afd83da78c","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:57.750Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"511aabef-0612-4d81-b3ef-60afd83da78c"}
{"level":"info","ts":"2026-09-18T09:50:57.750Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"511aabef-0612-4d81-b3ef-60afd83da78c","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:50:57.750Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"511aabef-0612-4d81-b3ef-60afd83da78c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:57.750Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"511aabef-0612-4d81-b3ef-60afd83da78c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:50:57.766Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"26c4d33a-ef1c-4256-a5fb-7328c52c4d31","name":"long-running-xrs5s-head"}
{"level":"info","ts":"2026-09-18T09:50:57.767Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4f3ce83f-28ed-4555-b239-e7ea6cae64af","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:57.767Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4f3ce83f-28ed-4555-b239-e7ea6cae64af","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:57.767Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4f3ce83f-28ed-4555-b239-e7ea6cae64af","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:57.768Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4f3ce83f-28ed-4555-b239-e7ea6cae64af"}
{"level":"info","ts":"2026-09-18T09:50:57.768Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4f3ce83f-28ed-4555-b239-e7ea6cae64af","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:50:57.768Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"4f3ce83f-28ed-4555-b239-e7ea6cae64af","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:50:57.778Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"00a35ec5-ea12-4505-8870-e3645f5dbf0c","certificate":"ray-head-cert-long-running-xrs5s","dnsNames":["long-running-xrs5s-head-svc","localhost","long-running-xrs5s-head-svc.test-ns-chvpt.svc","long-running-xrs5s-head-svc.test-ns-chvpt.svc.cluster.local"],"ipAddresses":["10.132.0.35","10.133.0.34","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:50:57.790Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"26c4d33a-ef1c-4256-a5fb-7328c52c4d31","name":"long-running-xrs5s-workers"}
{"level":"info","ts":"2026-09-18T09:50:57.790Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"26c4d33a-ef1c-4256-a5fb-7328c52c4d31","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:57.800Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"00a35ec5-ea12-4505-8870-e3645f5dbf0c","certificate":"ray-worker-cert-long-running-xrs5s","dnsNames":["long-running-xrs5s-worker-svc","localhost","long-running-xrs5s-small-group","long-running-xrs5s-worker-svc.test-ns-chvpt.svc","long-running-xrs5s-worker-svc.test-ns-chvpt.svc.cluster.local","*.long-running-xrs5s-worker-svc.test-ns-chvpt.svc","*.long-running-xrs5s-worker-svc.test-ns-chvpt.svc.cluster.local","*-worker-*.test-ns-chvpt.svc","*-worker-*.test-ns-chvpt.svc.cluster.local"],"ipAddresses":["10.132.0.35","10.133.0.34","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:50:57.800Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"00a35ec5-ea12-4505-8870-e3645f5dbf0c"}
{"level":"info","ts":"2026-09-18T09:50:58.572Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"0f6204e2-9fc2-4bd7-a42b-1d1d988df042"}
{"level":"info","ts":"2026-09-18T09:50:58.572Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"0f6204e2-9fc2-4bd7-a42b-1d1d988df042"}
{"level":"info","ts":"2026-09-18T09:50:58.572Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"0f6204e2-9fc2-4bd7-a42b-1d1d988df042"}
{"level":"info","ts":"2026-09-18T09:50:58.572Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"0f6204e2-9fc2-4bd7-a42b-1d1d988df042","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:50:58.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"0f6204e2-9fc2-4bd7-a42b-1d1d988df042","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:50:58.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"0f6204e2-9fc2-4bd7-a42b-1d1d988df042","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:58.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"0f6204e2-9fc2-4bd7-a42b-1d1d988df042","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:50:58.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"0f6204e2-9fc2-4bd7-a42b-1d1d988df042","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:50:58.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"0f6204e2-9fc2-4bd7-a42b-1d1d988df042","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:58.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"0f6204e2-9fc2-4bd7-a42b-1d1d988df042","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:50:58.572Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"0f6204e2-9fc2-4bd7-a42b-1d1d988df042","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:50:58.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"0f6204e2-9fc2-4bd7-a42b-1d1d988df042","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:50:58.572Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"0f6204e2-9fc2-4bd7-a42b-1d1d988df042","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:58.572Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"0f6204e2-9fc2-4bd7-a42b-1d1d988df042","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:50:58.572Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"0f6204e2-9fc2-4bd7-a42b-1d1d988df042","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:50:58.572Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"0f6204e2-9fc2-4bd7-a42b-1d1d988df042","seconds":300}
{"level":"info","ts":"2026-09-18T09:50:59.860Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"632f64ca-9b6f-4803-a377-b4f41df29851","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:50:59.860Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"632f64ca-9b6f-4803-a377-b4f41df29851","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:50:59.860Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"632f64ca-9b6f-4803-a377-b4f41df29851","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:50:59.860Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"632f64ca-9b6f-4803-a377-b4f41df29851"}
{"level":"info","ts":"2026-09-18T09:50:59.860Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"632f64ca-9b6f-4803-a377-b4f41df29851","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:50:59.860Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"632f64ca-9b6f-4803-a377-b4f41df29851","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:02.494Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"464e8366-7712-4f07-9441-96c765bfb9ff"}
{"level":"info","ts":"2026-09-18T09:51:02.494Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"464e8366-7712-4f07-9441-96c765bfb9ff"}
{"level":"info","ts":"2026-09-18T09:51:02.494Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"464e8366-7712-4f07-9441-96c765bfb9ff"}
{"level":"info","ts":"2026-09-18T09:51:02.494Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"464e8366-7712-4f07-9441-96c765bfb9ff","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:02.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"464e8366-7712-4f07-9441-96c765bfb9ff","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:02.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"464e8366-7712-4f07-9441-96c765bfb9ff","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:02.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"464e8366-7712-4f07-9441-96c765bfb9ff","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:02.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"464e8366-7712-4f07-9441-96c765bfb9ff","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:02.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"464e8366-7712-4f07-9441-96c765bfb9ff","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:02.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"464e8366-7712-4f07-9441-96c765bfb9ff","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:02.494Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"464e8366-7712-4f07-9441-96c765bfb9ff","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:02.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"464e8366-7712-4f07-9441-96c765bfb9ff","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:02.494Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"464e8366-7712-4f07-9441-96c765bfb9ff","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:02.494Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"464e8366-7712-4f07-9441-96c765bfb9ff","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:02.494Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"464e8366-7712-4f07-9441-96c765bfb9ff","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:51:02.494Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"464e8366-7712-4f07-9441-96c765bfb9ff","seconds":300}
{"level":"info","ts":"2026-09-18T09:51:02.746Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9edc98b0-52f6-4da5-9098-532539b739f1"}
{"level":"info","ts":"2026-09-18T09:51:02.746Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9edc98b0-52f6-4da5-9098-532539b739f1"}
{"level":"info","ts":"2026-09-18T09:51:02.746Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9edc98b0-52f6-4da5-9098-532539b739f1"}
{"level":"info","ts":"2026-09-18T09:51:02.746Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9edc98b0-52f6-4da5-9098-532539b739f1","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:02.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9edc98b0-52f6-4da5-9098-532539b739f1","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:02.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9edc98b0-52f6-4da5-9098-532539b739f1","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:02.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9edc98b0-52f6-4da5-9098-532539b739f1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:02.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9edc98b0-52f6-4da5-9098-532539b739f1","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:02.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9edc98b0-52f6-4da5-9098-532539b739f1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:02.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9edc98b0-52f6-4da5-9098-532539b739f1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:02.747Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9edc98b0-52f6-4da5-9098-532539b739f1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:02.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9edc98b0-52f6-4da5-9098-532539b739f1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:02.747Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9edc98b0-52f6-4da5-9098-532539b739f1","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:02.747Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9edc98b0-52f6-4da5-9098-532539b739f1","podName":"long-running-xrs5s-head-lcfg7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:02.747Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9edc98b0-52f6-4da5-9098-532539b739f1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:51:02.747Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9edc98b0-52f6-4da5-9098-532539b739f1","seconds":300}
{"level":"info","ts":"2026-09-18T09:51:02.860Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"730c6606-1ce0-42cb-9501-dc01723e0057","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:02.860Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"730c6606-1ce0-42cb-9501-dc01723e0057","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:02.860Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"730c6606-1ce0-42cb-9501-dc01723e0057","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:02.860Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"730c6606-1ce0-42cb-9501-dc01723e0057"}
{"level":"info","ts":"2026-09-18T09:51:02.860Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"730c6606-1ce0-42cb-9501-dc01723e0057","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:51:02.861Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"730c6606-1ce0-42cb-9501-dc01723e0057","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:03.499Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"09b62f2e-2ab5-4554-898c-a26ebc0be796"}
{"level":"info","ts":"2026-09-18T09:51:03.499Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"09b62f2e-2ab5-4554-898c-a26ebc0be796"}
{"level":"info","ts":"2026-09-18T09:51:03.499Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"09b62f2e-2ab5-4554-898c-a26ebc0be796"}
{"level":"info","ts":"2026-09-18T09:51:03.499Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"09b62f2e-2ab5-4554-898c-a26ebc0be796","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:03.500Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"09b62f2e-2ab5-4554-898c-a26ebc0be796","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:03.500Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"09b62f2e-2ab5-4554-898c-a26ebc0be796","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:03.500Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"09b62f2e-2ab5-4554-898c-a26ebc0be796","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:03.500Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"09b62f2e-2ab5-4554-898c-a26ebc0be796","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:03.500Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"09b62f2e-2ab5-4554-898c-a26ebc0be796","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:03.500Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"09b62f2e-2ab5-4554-898c-a26ebc0be796","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:03.500Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"09b62f2e-2ab5-4554-898c-a26ebc0be796","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:03.500Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"09b62f2e-2ab5-4554-898c-a26ebc0be796","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:03.500Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"09b62f2e-2ab5-4554-898c-a26ebc0be796","podName":"long-running-xrs5s-small-group-worker-cqdn2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:03.500Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"09b62f2e-2ab5-4554-898c-a26ebc0be796","podName":"long-running-xrs5s-small-group-worker-cqdn2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:03.500Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"09b62f2e-2ab5-4554-898c-a26ebc0be796","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:51:03.500Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"09b62f2e-2ab5-4554-898c-a26ebc0be796","seconds":300}
{"level":"info","ts":"2026-09-18T09:51:03.759Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"f61fa26e-a759-47fe-aecf-bcc5a9ca971f"}
{"level":"info","ts":"2026-09-18T09:51:03.759Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"f61fa26e-a759-47fe-aecf-bcc5a9ca971f"}
{"level":"info","ts":"2026-09-18T09:51:03.759Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"f61fa26e-a759-47fe-aecf-bcc5a9ca971f"}
{"level":"info","ts":"2026-09-18T09:51:03.759Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"f61fa26e-a759-47fe-aecf-bcc5a9ca971f","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:03.759Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"f61fa26e-a759-47fe-aecf-bcc5a9ca971f","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:03.759Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"f61fa26e-a759-47fe-aecf-bcc5a9ca971f","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:03.759Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"f61fa26e-a759-47fe-aecf-bcc5a9ca971f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:03.759Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"f61fa26e-a759-47fe-aecf-bcc5a9ca971f","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:03.759Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"f61fa26e-a759-47fe-aecf-bcc5a9ca971f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:03.759Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"f61fa26e-a759-47fe-aecf-bcc5a9ca971f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:03.759Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"f61fa26e-a759-47fe-aecf-bcc5a9ca971f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:03.759Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"f61fa26e-a759-47fe-aecf-bcc5a9ca971f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:03.759Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"f61fa26e-a759-47fe-aecf-bcc5a9ca971f","podName":"long-running-xrs5s-small-group-worker-cqdn2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:03.759Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"f61fa26e-a759-47fe-aecf-bcc5a9ca971f","podName":"long-running-xrs5s-small-group-worker-cqdn2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:03.759Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"f61fa26e-a759-47fe-aecf-bcc5a9ca971f","name":"long-running-xrs5s","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:51:03.780Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"fe44f98a-aeb4-405c-a7c1-81a4de557f69","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:03.780Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"d3a5aef5-1945-4157-9fdb-a3378a77e347","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:03.781Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"fe44f98a-aeb4-405c-a7c1-81a4de557f69","pod":"long-running-xrs5s-small-group-worker-cqdn2","ip":"10.133.0.34"}
{"level":"info","ts":"2026-09-18T09:51:03.781Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"fe44f98a-aeb4-405c-a7c1-81a4de557f69","pod":"long-running-xrs5s-head-lcfg7","ip":"10.132.0.35"}
{"level":"info","ts":"2026-09-18T09:51:03.781Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"fe44f98a-aeb4-405c-a7c1-81a4de557f69","rayCluster":"long-running-xrs5s","podIPs":["10.133.0.34","10.132.0.35"]}
{"level":"info","ts":"2026-09-18T09:51:03.781Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d22437a5-c91b-41f3-a62a-5ae62f84bd90","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:03.781Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d22437a5-c91b-41f3-a62a-5ae62f84bd90","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:03.781Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d22437a5-c91b-41f3-a62a-5ae62f84bd90","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:03.781Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d22437a5-c91b-41f3-a62a-5ae62f84bd90"}
{"level":"info","ts":"2026-09-18T09:51:03.781Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d22437a5-c91b-41f3-a62a-5ae62f84bd90","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:51:03.781Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d22437a5-c91b-41f3-a62a-5ae62f84bd90","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:50:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:03.781Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"d22437a5-c91b-41f3-a62a-5ae62f84bd90","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:51:03.798Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"3c322669-c8ff-4d90-bdde-cb9e0a17226c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:03.798Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"3c322669-c8ff-4d90-bdde-cb9e0a17226c","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:03.798Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"3c322669-c8ff-4d90-bdde-cb9e0a17226c","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:03.799Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"3c322669-c8ff-4d90-bdde-cb9e0a17226c"}
{"level":"info","ts":"2026-09-18T09:51:03.799Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"3c322669-c8ff-4d90-bdde-cb9e0a17226c","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:51:03.799Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"3c322669-c8ff-4d90-bdde-cb9e0a17226c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:03.804Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"d3a5aef5-1945-4157-9fdb-a3378a77e347","name":"long-running-xrs5s-head"}
{"level":"info","ts":"2026-09-18T09:51:03.816Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"d3a5aef5-1945-4157-9fdb-a3378a77e347","name":"long-running-xrs5s-workers"}
{"level":"info","ts":"2026-09-18T09:51:03.816Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"d3a5aef5-1945-4157-9fdb-a3378a77e347","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:05.781Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9caf5428-ef4c-459c-8c60-51df3ea3c0bc"}
{"level":"info","ts":"2026-09-18T09:51:05.781Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9caf5428-ef4c-459c-8c60-51df3ea3c0bc"}
{"level":"info","ts":"2026-09-18T09:51:05.781Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9caf5428-ef4c-459c-8c60-51df3ea3c0bc"}
{"level":"info","ts":"2026-09-18T09:51:05.781Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9caf5428-ef4c-459c-8c60-51df3ea3c0bc","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:05.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9caf5428-ef4c-459c-8c60-51df3ea3c0bc","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:05.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9caf5428-ef4c-459c-8c60-51df3ea3c0bc","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:05.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9caf5428-ef4c-459c-8c60-51df3ea3c0bc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:05.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9caf5428-ef4c-459c-8c60-51df3ea3c0bc","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:05.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9caf5428-ef4c-459c-8c60-51df3ea3c0bc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:05.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9caf5428-ef4c-459c-8c60-51df3ea3c0bc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:05.781Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9caf5428-ef4c-459c-8c60-51df3ea3c0bc","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:05.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9caf5428-ef4c-459c-8c60-51df3ea3c0bc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:05.781Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9caf5428-ef4c-459c-8c60-51df3ea3c0bc","podName":"long-running-xrs5s-head-lcfg7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:50:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:51:05.781Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9caf5428-ef4c-459c-8c60-51df3ea3c0bc","podName":"long-running-xrs5s-head-lcfg7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:50:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:51:05.781Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9caf5428-ef4c-459c-8c60-51df3ea3c0bc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:51:05.781Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"9caf5428-ef4c-459c-8c60-51df3ea3c0bc","seconds":300}
{"level":"info","ts":"2026-09-18T09:51:05.861Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"477e6db7-7fc0-4cdf-8050-8869e9852e3c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:05.861Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"477e6db7-7fc0-4cdf-8050-8869e9852e3c","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:05.861Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"477e6db7-7fc0-4cdf-8050-8869e9852e3c","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:05.861Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"477e6db7-7fc0-4cdf-8050-8869e9852e3c"}
{"level":"info","ts":"2026-09-18T09:51:05.861Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"477e6db7-7fc0-4cdf-8050-8869e9852e3c","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:51:05.861Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"477e6db7-7fc0-4cdf-8050-8869e9852e3c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:08.862Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c8dcc3e3-5e58-4510-8f0f-85da24cfa98f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:08.862Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c8dcc3e3-5e58-4510-8f0f-85da24cfa98f","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:08.862Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c8dcc3e3-5e58-4510-8f0f-85da24cfa98f","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:08.862Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c8dcc3e3-5e58-4510-8f0f-85da24cfa98f"}
{"level":"info","ts":"2026-09-18T09:51:08.862Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c8dcc3e3-5e58-4510-8f0f-85da24cfa98f","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:51:08.862Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"c8dcc3e3-5e58-4510-8f0f-85da24cfa98f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:11.863Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7b2a5d3d-056a-4026-803d-ed4ac596737e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:11.863Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7b2a5d3d-056a-4026-803d-ed4ac596737e","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:11.863Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7b2a5d3d-056a-4026-803d-ed4ac596737e","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:11.863Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7b2a5d3d-056a-4026-803d-ed4ac596737e"}
{"level":"info","ts":"2026-09-18T09:51:11.863Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7b2a5d3d-056a-4026-803d-ed4ac596737e","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:51:11.863Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"7b2a5d3d-056a-4026-803d-ed4ac596737e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:12.544Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8be9542b-d8cb-423a-817c-4564be1ade6c"}
{"level":"info","ts":"2026-09-18T09:51:12.544Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8be9542b-d8cb-423a-817c-4564be1ade6c"}
{"level":"info","ts":"2026-09-18T09:51:12.544Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8be9542b-d8cb-423a-817c-4564be1ade6c"}
{"level":"info","ts":"2026-09-18T09:51:12.544Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8be9542b-d8cb-423a-817c-4564be1ade6c","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:12.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8be9542b-d8cb-423a-817c-4564be1ade6c","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:12.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8be9542b-d8cb-423a-817c-4564be1ade6c","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:12.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8be9542b-d8cb-423a-817c-4564be1ade6c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:12.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8be9542b-d8cb-423a-817c-4564be1ade6c","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:12.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8be9542b-d8cb-423a-817c-4564be1ade6c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:12.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8be9542b-d8cb-423a-817c-4564be1ade6c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:12.544Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8be9542b-d8cb-423a-817c-4564be1ade6c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:12.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8be9542b-d8cb-423a-817c-4564be1ade6c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:12.545Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8be9542b-d8cb-423a-817c-4564be1ade6c","podName":"long-running-xrs5s-head-lcfg7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:50:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:51:12.545Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8be9542b-d8cb-423a-817c-4564be1ade6c","podName":"long-running-xrs5s-head-lcfg7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:50:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:51:12.545Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8be9542b-d8cb-423a-817c-4564be1ade6c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:51:12.545Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"8be9542b-d8cb-423a-817c-4564be1ade6c","seconds":300}
{"level":"info","ts":"2026-09-18T09:51:13.554Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4baa5fa0-87e8-4d59-a96e-433f2bcbe956"}
{"level":"info","ts":"2026-09-18T09:51:13.554Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4baa5fa0-87e8-4d59-a96e-433f2bcbe956"}
{"level":"info","ts":"2026-09-18T09:51:13.554Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4baa5fa0-87e8-4d59-a96e-433f2bcbe956"}
{"level":"info","ts":"2026-09-18T09:51:13.554Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4baa5fa0-87e8-4d59-a96e-433f2bcbe956","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:13.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4baa5fa0-87e8-4d59-a96e-433f2bcbe956","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:13.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4baa5fa0-87e8-4d59-a96e-433f2bcbe956","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:13.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4baa5fa0-87e8-4d59-a96e-433f2bcbe956","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:13.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4baa5fa0-87e8-4d59-a96e-433f2bcbe956","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:13.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4baa5fa0-87e8-4d59-a96e-433f2bcbe956","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:13.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4baa5fa0-87e8-4d59-a96e-433f2bcbe956","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:13.555Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4baa5fa0-87e8-4d59-a96e-433f2bcbe956","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:13.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4baa5fa0-87e8-4d59-a96e-433f2bcbe956","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:13.555Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4baa5fa0-87e8-4d59-a96e-433f2bcbe956","podName":"long-running-xrs5s-head-lcfg7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:50:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:51:13.555Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4baa5fa0-87e8-4d59-a96e-433f2bcbe956","podName":"long-running-xrs5s-head-lcfg7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:50:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:51:13.556Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4baa5fa0-87e8-4d59-a96e-433f2bcbe956","name":"long-running-xrs5s","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:51:13.572Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"cd33aa06-61f7-4788-ae25-7f31cc797eae","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:13.572Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"cd33aa06-61f7-4788-ae25-7f31cc797eae","pod":"long-running-xrs5s-head-lcfg7","ip":"10.132.0.35"}
{"level":"info","ts":"2026-09-18T09:51:13.572Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"92c35920-a1f2-4f60-a0ee-9e8049739311","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:13.572Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"92c35920-a1f2-4f60-a0ee-9e8049739311","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:13.572Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"92c35920-a1f2-4f60-a0ee-9e8049739311","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:13.572Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"f32f2a99-eb6c-4459-9033-cc188f6922a8","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:13.572Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"92c35920-a1f2-4f60-a0ee-9e8049739311"}
{"level":"info","ts":"2026-09-18T09:51:13.572Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"92c35920-a1f2-4f60-a0ee-9e8049739311","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:51:13.572Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"92c35920-a1f2-4f60-a0ee-9e8049739311","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:13.572Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"92c35920-a1f2-4f60-a0ee-9e8049739311","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:51:13.572Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"cd33aa06-61f7-4788-ae25-7f31cc797eae","pod":"long-running-xrs5s-small-group-worker-cqdn2","ip":"10.133.0.34"}
{"level":"info","ts":"2026-09-18T09:51:13.573Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"cd33aa06-61f7-4788-ae25-7f31cc797eae","rayCluster":"long-running-xrs5s","podIPs":["10.132.0.35","10.133.0.34"]}
{"level":"info","ts":"2026-09-18T09:51:13.586Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"f32f2a99-eb6c-4459-9033-cc188f6922a8","name":"long-running-xrs5s-head"}
{"level":"info","ts":"2026-09-18T09:51:13.666Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b41d1a87-e3c2-4413-8847-70c55d4166b3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:13.666Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b41d1a87-e3c2-4413-8847-70c55d4166b3","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:13.666Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b41d1a87-e3c2-4413-8847-70c55d4166b3","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:13.666Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b41d1a87-e3c2-4413-8847-70c55d4166b3"}
{"level":"info","ts":"2026-09-18T09:51:13.666Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b41d1a87-e3c2-4413-8847-70c55d4166b3","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:51:13.666Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b41d1a87-e3c2-4413-8847-70c55d4166b3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:13.675Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"f32f2a99-eb6c-4459-9033-cc188f6922a8","name":"long-running-xrs5s-workers"}
{"level":"info","ts":"2026-09-18T09:51:13.675Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"f32f2a99-eb6c-4459-9033-cc188f6922a8","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:14.787Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"e98e11dc-8861-4a9f-9560-3ca06ee81e3f"}
{"level":"info","ts":"2026-09-18T09:51:14.787Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"e98e11dc-8861-4a9f-9560-3ca06ee81e3f"}
{"level":"info","ts":"2026-09-18T09:51:14.787Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"e98e11dc-8861-4a9f-9560-3ca06ee81e3f"}
{"level":"info","ts":"2026-09-18T09:51:14.787Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"e98e11dc-8861-4a9f-9560-3ca06ee81e3f","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:14.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"e98e11dc-8861-4a9f-9560-3ca06ee81e3f","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:14.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"e98e11dc-8861-4a9f-9560-3ca06ee81e3f","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:14.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"e98e11dc-8861-4a9f-9560-3ca06ee81e3f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:14.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"e98e11dc-8861-4a9f-9560-3ca06ee81e3f","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:14.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"e98e11dc-8861-4a9f-9560-3ca06ee81e3f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:14.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"e98e11dc-8861-4a9f-9560-3ca06ee81e3f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:14.788Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"e98e11dc-8861-4a9f-9560-3ca06ee81e3f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:14.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"e98e11dc-8861-4a9f-9560-3ca06ee81e3f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:14.788Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"e98e11dc-8861-4a9f-9560-3ca06ee81e3f","podName":"long-running-xrs5s-small-group-worker-cqdn2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:50:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:51:14.788Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"e98e11dc-8861-4a9f-9560-3ca06ee81e3f","podName":"long-running-xrs5s-small-group-worker-cqdn2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:50:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:51:14.788Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"e98e11dc-8861-4a9f-9560-3ca06ee81e3f","name":"long-running-xrs5s","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:51:14.806Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"04f65764-f2c7-48d8-af5c-d3c362ed095a","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:14.806Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"cc82f351-1d78-47f0-825a-df14828713c0","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:14.806Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"04f65764-f2c7-48d8-af5c-d3c362ed095a","pod":"long-running-xrs5s-head-lcfg7","ip":"10.132.0.35"}
{"level":"info","ts":"2026-09-18T09:51:14.806Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"04f65764-f2c7-48d8-af5c-d3c362ed095a","pod":"long-running-xrs5s-small-group-worker-cqdn2","ip":"10.133.0.34"}
{"level":"info","ts":"2026-09-18T09:51:14.806Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"04f65764-f2c7-48d8-af5c-d3c362ed095a","rayCluster":"long-running-xrs5s","podIPs":["10.132.0.35","10.133.0.34"]}
{"level":"info","ts":"2026-09-18T09:51:14.806Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2979c249-8ab7-47b2-8f4d-274f14403662","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:14.806Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2979c249-8ab7-47b2-8f4d-274f14403662","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:14.806Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2979c249-8ab7-47b2-8f4d-274f14403662","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:14.806Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2979c249-8ab7-47b2-8f4d-274f14403662"}
{"level":"info","ts":"2026-09-18T09:51:14.806Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2979c249-8ab7-47b2-8f4d-274f14403662","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:51:14.806Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2979c249-8ab7-47b2-8f4d-274f14403662","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:14.806Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2979c249-8ab7-47b2-8f4d-274f14403662","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:51:14.820Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"cc82f351-1d78-47f0-825a-df14828713c0","name":"long-running-xrs5s-head"}
{"level":"info","ts":"2026-09-18T09:51:14.822Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2717607f-45a9-44c9-89a1-22fc7d8874f7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:14.822Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2717607f-45a9-44c9-89a1-22fc7d8874f7","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:14.822Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2717607f-45a9-44c9-89a1-22fc7d8874f7","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:14.822Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2717607f-45a9-44c9-89a1-22fc7d8874f7"}
{"level":"info","ts":"2026-09-18T09:51:14.822Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2717607f-45a9-44c9-89a1-22fc7d8874f7","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:51:14.822Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2717607f-45a9-44c9-89a1-22fc7d8874f7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:14.833Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"cc82f351-1d78-47f0-825a-df14828713c0","name":"long-running-xrs5s-workers"}
{"level":"info","ts":"2026-09-18T09:51:14.833Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"cc82f351-1d78-47f0-825a-df14828713c0","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:14.864Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b6e8dc5b-8166-4ac8-adc6-d928112ae30b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:14.864Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b6e8dc5b-8166-4ac8-adc6-d928112ae30b","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:14.864Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b6e8dc5b-8166-4ac8-adc6-d928112ae30b","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:14.864Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b6e8dc5b-8166-4ac8-adc6-d928112ae30b"}
{"level":"info","ts":"2026-09-18T09:51:14.864Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b6e8dc5b-8166-4ac8-adc6-d928112ae30b","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:51:14.864Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b6e8dc5b-8166-4ac8-adc6-d928112ae30b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:15.574Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"088c0027-54a0-4fc9-b19c-95a603df5c40"}
{"level":"info","ts":"2026-09-18T09:51:15.574Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"088c0027-54a0-4fc9-b19c-95a603df5c40"}
{"level":"info","ts":"2026-09-18T09:51:15.574Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"088c0027-54a0-4fc9-b19c-95a603df5c40"}
{"level":"info","ts":"2026-09-18T09:51:15.574Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"088c0027-54a0-4fc9-b19c-95a603df5c40","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:15.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"088c0027-54a0-4fc9-b19c-95a603df5c40","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:15.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"088c0027-54a0-4fc9-b19c-95a603df5c40","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:15.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"088c0027-54a0-4fc9-b19c-95a603df5c40","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:15.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"088c0027-54a0-4fc9-b19c-95a603df5c40","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:15.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"088c0027-54a0-4fc9-b19c-95a603df5c40","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:15.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"088c0027-54a0-4fc9-b19c-95a603df5c40","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:15.574Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"088c0027-54a0-4fc9-b19c-95a603df5c40","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:15.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"088c0027-54a0-4fc9-b19c-95a603df5c40","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:15.574Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"088c0027-54a0-4fc9-b19c-95a603df5c40","podName":"long-running-xrs5s-small-group-worker-cqdn2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:50:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:51:15.574Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"088c0027-54a0-4fc9-b19c-95a603df5c40","podName":"long-running-xrs5s-small-group-worker-cqdn2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:50:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:51:15.574Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"088c0027-54a0-4fc9-b19c-95a603df5c40","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:51:15.574Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"088c0027-54a0-4fc9-b19c-95a603df5c40","seconds":300}
{"level":"info","ts":"2026-09-18T09:51:17.865Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"31918ae2-7aa9-4e2a-a906-a88a66a1ce91","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:17.865Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"31918ae2-7aa9-4e2a-a906-a88a66a1ce91","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:17.865Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"31918ae2-7aa9-4e2a-a906-a88a66a1ce91","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:17.865Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"31918ae2-7aa9-4e2a-a906-a88a66a1ce91"}
{"level":"info","ts":"2026-09-18T09:51:17.865Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"31918ae2-7aa9-4e2a-a906-a88a66a1ce91","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:51:17.865Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"31918ae2-7aa9-4e2a-a906-a88a66a1ce91","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:20.866Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"984fd4e5-a183-4a4e-9f8f-a99247f222de","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:20.866Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"984fd4e5-a183-4a4e-9f8f-a99247f222de","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:20.866Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"984fd4e5-a183-4a4e-9f8f-a99247f222de","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:20.866Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"984fd4e5-a183-4a4e-9f8f-a99247f222de"}
{"level":"info","ts":"2026-09-18T09:51:20.866Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"984fd4e5-a183-4a4e-9f8f-a99247f222de","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:51:20.866Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"984fd4e5-a183-4a4e-9f8f-a99247f222de","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:23.867Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"66db47e9-d847-4952-945e-7e84c6bf4551","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:23.867Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"66db47e9-d847-4952-945e-7e84c6bf4551","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:23.867Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"66db47e9-d847-4952-945e-7e84c6bf4551","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:23.867Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"66db47e9-d847-4952-945e-7e84c6bf4551"}
{"level":"info","ts":"2026-09-18T09:51:23.867Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"66db47e9-d847-4952-945e-7e84c6bf4551","RayCluster":"long-running-xrs5s","State":""}
{"level":"info","ts":"2026-09-18T09:51:23.867Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"66db47e9-d847-4952-945e-7e84c6bf4551","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:24.087Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"2c3048ea-3e3e-44ac-91f8-dfbf388c7a2f","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:24.087Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"2c3048ea-3e3e-44ac-91f8-dfbf388c7a2f","pod":"long-running-xrs5s-head-lcfg7","ip":"10.132.0.35"}
{"level":"info","ts":"2026-09-18T09:51:24.087Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"2c3048ea-3e3e-44ac-91f8-dfbf388c7a2f","pod":"long-running-xrs5s-small-group-worker-cqdn2","ip":"10.133.0.34"}
{"level":"info","ts":"2026-09-18T09:51:24.087Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"2c3048ea-3e3e-44ac-91f8-dfbf388c7a2f","rayCluster":"long-running-xrs5s","podIPs":["10.132.0.35","10.133.0.34"]}
{"level":"info","ts":"2026-09-18T09:51:24.580Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4660d689-80d2-49cf-92e1-b8f9bcca3779"}
{"level":"info","ts":"2026-09-18T09:51:24.580Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4660d689-80d2-49cf-92e1-b8f9bcca3779"}
{"level":"info","ts":"2026-09-18T09:51:24.580Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4660d689-80d2-49cf-92e1-b8f9bcca3779"}
{"level":"info","ts":"2026-09-18T09:51:24.580Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4660d689-80d2-49cf-92e1-b8f9bcca3779","1 head service found":"long-running-xrs5s-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:24.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4660d689-80d2-49cf-92e1-b8f9bcca3779","Found 1 head Pod":"long-running-xrs5s-head-lcfg7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:24.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4660d689-80d2-49cf-92e1-b8f9bcca3779","head Pod":"long-running-xrs5s-head-lcfg7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-xrs5s-head-lcfg7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:24.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4660d689-80d2-49cf-92e1-b8f9bcca3779","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:24.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4660d689-80d2-49cf-92e1-b8f9bcca3779","worker Pod":"long-running-xrs5s-small-group-worker-cqdn2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-xrs5s-small-group-worker-cqdn2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:24.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4660d689-80d2-49cf-92e1-b8f9bcca3779","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:24.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4660d689-80d2-49cf-92e1-b8f9bcca3779","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:24.580Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4660d689-80d2-49cf-92e1-b8f9bcca3779","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:24.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4660d689-80d2-49cf-92e1-b8f9bcca3779","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:24.580Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"4660d689-80d2-49cf-92e1-b8f9bcca3779","name":"long-running-xrs5s","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:24Z","stateTransitionTimes":{"ready":"2026-09-18T09:51:24Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:51:24Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:51:24.600Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"c3af23d2-ae64-4cd2-a990-0febd962a741","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:24.600Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"31672dfd-983c-4797-8499-7c595779a47f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:24.600Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"31672dfd-983c-4797-8499-7c595779a47f","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:24.600Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"31672dfd-983c-4797-8499-7c595779a47f","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:24.600Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"31672dfd-983c-4797-8499-7c595779a47f"}
{"level":"info","ts":"2026-09-18T09:51:24.600Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"21259742-98e1-47fc-8d41-472a56179c6f","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:24.600Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"21259742-98e1-47fc-8d41-472a56179c6f","pod":"long-running-xrs5s-head-lcfg7","ip":"10.132.0.35"}
{"level":"info","ts":"2026-09-18T09:51:24.600Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"21259742-98e1-47fc-8d41-472a56179c6f","pod":"long-running-xrs5s-small-group-worker-cqdn2","ip":"10.133.0.34"}
{"level":"info","ts":"2026-09-18T09:51:24.600Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"21259742-98e1-47fc-8d41-472a56179c6f","rayCluster":"long-running-xrs5s","podIPs":["10.132.0.35","10.133.0.34"]}
{"level":"info","ts":"2026-09-18T09:51:24.617Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"31672dfd-983c-4797-8499-7c595779a47f","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-09-18T09:51:24.617Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"31672dfd-983c-4797-8499-7c595779a47f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7slts","rayClusterName":"long-running-xrs5s","dashboardURL":"long-running-xrs5s-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:50:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-xrs5s-head-lcfg7","serviceName":"long-running-xrs5s-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:50:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:50:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:24.617Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"31672dfd-983c-4797-8499-7c595779a47f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:51:24.633Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"c3af23d2-ae64-4cd2-a990-0febd962a741","name":"long-running-xrs5s-head"}
{"level":"info","ts":"2026-09-18T09:51:24.643Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e7afc222-ec91-43e4-bac5-f200195468aa","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:24.643Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e7afc222-ec91-43e4-bac5-f200195468aa","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:24.643Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e7afc222-ec91-43e4-bac5-f200195468aa","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:24.643Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e7afc222-ec91-43e4-bac5-f200195468aa"}
{"level":"info","ts":"2026-09-18T09:51:24.643Z","logger":"controllers.RayJob","msg":"Update Kubernetes Service","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e7afc222-ec91-43e4-bac5-f200195468aa","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:51:24.675Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"c3af23d2-ae64-4cd2-a990-0febd962a741","name":"long-running-xrs5s-workers"}
{"level":"info","ts":"2026-09-18T09:51:24.675Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"c3af23d2-ae64-4cd2-a990-0febd962a741","cluster":"long-running-xrs5s"}
{"level":"error","ts":"2026-09-18T09:51:24.678Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e7afc222-ec91-43e4-bac5-f200195468aa","JobId":"long-running-7slts","error":"Job long-running-7slts does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:24.678Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e7afc222-ec91-43e4-bac5-f200195468aa"}
{"level":"error","ts":"2026-09-18T09:51:24.678Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"e7afc222-ec91-43e4-bac5-f200195468aa","error":"Job long-running-7slts does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:24.678Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"059e4c16-04de-497a-a94c-303868795c80","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:24.678Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"059e4c16-04de-497a-a94c-303868795c80","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:24.678Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"059e4c16-04de-497a-a94c-303868795c80","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:24.678Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"059e4c16-04de-497a-a94c-303868795c80"}
{"level":"info","ts":"2026-09-18T09:51:24.678Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"059e4c16-04de-497a-a94c-303868795c80","rayCluster":"long-running-xrs5s","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:51:24.683Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"059e4c16-04de-497a-a94c-303868795c80","JobId":"long-running-7slts","error":"Job long-running-7slts does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:24.683Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"059e4c16-04de-497a-a94c-303868795c80"}
{"level":"error","ts":"2026-09-18T09:51:24.683Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"059e4c16-04de-497a-a94c-303868795c80","error":"Job long-running-7slts does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:24.683Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"fbc1f70b-e251-4f4e-ae74-9630dbb10c57","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:24.683Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"fbc1f70b-e251-4f4e-ae74-9630dbb10c57","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:24.683Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"fbc1f70b-e251-4f4e-ae74-9630dbb10c57","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:24.684Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"fbc1f70b-e251-4f4e-ae74-9630dbb10c57"}
{"level":"info","ts":"2026-09-18T09:51:24.684Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"fbc1f70b-e251-4f4e-ae74-9630dbb10c57","rayCluster":"long-running-xrs5s","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:51:24.688Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"fbc1f70b-e251-4f4e-ae74-9630dbb10c57","JobId":"long-running-7slts","error":"Job long-running-7slts does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:24.688Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"fbc1f70b-e251-4f4e-ae74-9630dbb10c57"}
{"level":"error","ts":"2026-09-18T09:51:24.688Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"fbc1f70b-e251-4f4e-ae74-9630dbb10c57","error":"Job long-running-7slts does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:24.694Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"5c8ae6c2-89cd-4cd7-8063-df42baa048de","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:24.694Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"5c8ae6c2-89cd-4cd7-8063-df42baa048de","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:24.694Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"5c8ae6c2-89cd-4cd7-8063-df42baa048de","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:24.695Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"5c8ae6c2-89cd-4cd7-8063-df42baa048de"}
{"level":"info","ts":"2026-09-18T09:51:24.695Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"5c8ae6c2-89cd-4cd7-8063-df42baa048de","rayCluster":"long-running-xrs5s","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:51:24.699Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"5c8ae6c2-89cd-4cd7-8063-df42baa048de","JobId":"long-running-7slts","error":"Job long-running-7slts does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:24.699Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"5c8ae6c2-89cd-4cd7-8063-df42baa048de"}
{"level":"error","ts":"2026-09-18T09:51:24.699Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"5c8ae6c2-89cd-4cd7-8063-df42baa048de","error":"Job long-running-7slts does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:24.704Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ce49eac6-943a-4f07-ab50-b9fa964d855b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:24.704Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ce49eac6-943a-4f07-ab50-b9fa964d855b","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:24.704Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ce49eac6-943a-4f07-ab50-b9fa964d855b","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:24.704Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ce49eac6-943a-4f07-ab50-b9fa964d855b"}
{"level":"info","ts":"2026-09-18T09:51:24.704Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ce49eac6-943a-4f07-ab50-b9fa964d855b","rayCluster":"long-running-xrs5s","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:51:24.709Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ce49eac6-943a-4f07-ab50-b9fa964d855b","JobId":"long-running-7slts","error":"Job long-running-7slts does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:24.709Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ce49eac6-943a-4f07-ab50-b9fa964d855b"}
{"level":"error","ts":"2026-09-18T09:51:24.709Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ce49eac6-943a-4f07-ab50-b9fa964d855b","error":"Job long-running-7slts does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:24.740Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"6002a921-30fe-45c6-8a6e-e6aeb1397496","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:24.740Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"6002a921-30fe-45c6-8a6e-e6aeb1397496","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:24.740Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"6002a921-30fe-45c6-8a6e-e6aeb1397496","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:24.740Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"6002a921-30fe-45c6-8a6e-e6aeb1397496"}
{"level":"info","ts":"2026-09-18T09:51:24.740Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"6002a921-30fe-45c6-8a6e-e6aeb1397496","rayCluster":"long-running-xrs5s","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:51:24.770Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"6002a921-30fe-45c6-8a6e-e6aeb1397496","JobId":"long-running-7slts","error":"Job long-running-7slts does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:24.770Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"6002a921-30fe-45c6-8a6e-e6aeb1397496"}
{"level":"error","ts":"2026-09-18T09:51:24.770Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"6002a921-30fe-45c6-8a6e-e6aeb1397496","error":"Job long-running-7slts does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:24.930Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"700ca90b-69ed-42bb-9b28-8e3349301249","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:24.930Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"700ca90b-69ed-42bb-9b28-8e3349301249","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:24.930Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"700ca90b-69ed-42bb-9b28-8e3349301249","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:24.930Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"700ca90b-69ed-42bb-9b28-8e3349301249"}
{"level":"info","ts":"2026-09-18T09:51:24.931Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"700ca90b-69ed-42bb-9b28-8e3349301249","rayCluster":"long-running-xrs5s","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:51:24.936Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"700ca90b-69ed-42bb-9b28-8e3349301249","JobId":"long-running-7slts","error":"Job long-running-7slts does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:24.936Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"700ca90b-69ed-42bb-9b28-8e3349301249"}
{"level":"error","ts":"2026-09-18T09:51:24.936Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"700ca90b-69ed-42bb-9b28-8e3349301249","error":"Job long-running-7slts does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:25.111Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2f7779f7-0d59-44f3-987f-fc7395a44cad","DeletionTimestamp":"2026-09-18 09:51:25 +0000 UTC"}
{"level":"error","ts":"2026-09-18T09:51:25.116Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2f7779f7-0d59-44f3-987f-fc7395a44cad","error":"invalid character 'J' looking for beginning of value","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:25.117Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"2f7779f7-0d59-44f3-987f-fc7395a44cad","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:51:25.150Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"da3e9017-4125-475f-a5b8-e1c2ee170119"}
{"level":"info","ts":"2026-09-18T09:51:25.152Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e9656712-5634-40a3-a721-c19ddef25b18","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:25.152Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e9656712-5634-40a3-a721-c19ddef25b18","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:51:25.173Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e9656712-5634-40a3-a721-c19ddef25b18"}
{"level":"info","ts":"2026-09-18T09:51:25.173Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e9656712-5634-40a3-a721-c19ddef25b18","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:51:25.173Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e9656712-5634-40a3-a721-c19ddef25b18","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kdp4b","rayClusterName":"counter-bstjf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:51:25.173Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e9656712-5634-40a3-a721-c19ddef25b18","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:51:25.196Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"83cb93a6-8a48-4ba8-b160-1bb01931acf3"}
{"level":"info","ts":"2026-09-18T09:51:25.196Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ef3c3a7b-b999-4d79-9794-8fb6a6dc57cf","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:25.196Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ef3c3a7b-b999-4d79-9794-8fb6a6dc57cf"}
{"level":"info","ts":"2026-09-18T09:51:25.196Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ef3c3a7b-b999-4d79-9794-8fb6a6dc57cf","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:51:25.196Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ef3c3a7b-b999-4d79-9794-8fb6a6dc57cf","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4fm8p","rayClusterName":"counter-mkgk4","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:51:25.196Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ef3c3a7b-b999-4d79-9794-8fb6a6dc57cf","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:51:25.208Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"e8f6a155-bde4-4be0-96ae-3fd0e3acbd3c","namespacedName":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:25.208Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"e8f6a155-bde4-4be0-96ae-3fd0e3acbd3c","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:25.208Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"e8f6a155-bde4-4be0-96ae-3fd0e3acbd3c","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:25.208Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"e8f6a155-bde4-4be0-96ae-3fd0e3acbd3c","namespace":"test-ns-chvpt","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T09:51:25.208Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"reconcileID":"b33b58c0-7b77-4825-9061-47d1e30eb47b"}
{"level":"info","ts":"2026-09-18T09:51:25.208Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"0ffbd200-93f1-46df-9054-98fc3b2d02d8","rayCluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:25.208Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"73c15537-b886-452f-9475-b9c78a44ced2"}
{"level":"info","ts":"2026-09-18T09:51:25.208Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"0ffbd200-93f1-46df-9054-98fc3b2d02d8","pod":"long-running-xrs5s-head-lcfg7","ip":"10.132.0.35"}
{"level":"info","ts":"2026-09-18T09:51:25.208Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"0ffbd200-93f1-46df-9054-98fc3b2d02d8","pod":"long-running-xrs5s-small-group-worker-cqdn2","ip":"10.133.0.34"}
{"level":"info","ts":"2026-09-18T09:51:25.208Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"0ffbd200-93f1-46df-9054-98fc3b2d02d8","rayCluster":"long-running-xrs5s","podIPs":["10.132.0.35","10.133.0.34"]}
{"level":"info","ts":"2026-09-18T09:51:25.220Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ef3c3a7b-b999-4d79-9794-8fb6a6dc57cf","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T09:51:25.220Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ef3c3a7b-b999-4d79-9794-8fb6a6dc57cf"}
{"level":"error","ts":"2026-09-18T09:51:25.220Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ef3c3a7b-b999-4d79-9794-8fb6a6dc57cf","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:25.220Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"ab2b2a2d-d400-4d69-a01d-277e2c256590"}
{"level":"info","ts":"2026-09-18T09:51:25.220Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c051828f-05cb-4cfe-bbc8-56a464dd6e4d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:25.220Z","logger":"controllers.RayJob","msg":"Try to transition the status to `Suspending`","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c051828f-05cb-4cfe-bbc8-56a464dd6e4d","oldStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:51:25.220Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c051828f-05cb-4cfe-bbc8-56a464dd6e4d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kdp4b","rayClusterName":"counter-bstjf","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kdp4b","rayClusterName":"counter-bstjf","jobDeploymentStatus":"Suspending","startTime":"2026-09-18T09:51:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:51:25.220Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c051828f-05cb-4cfe-bbc8-56a464dd6e4d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Suspending"}
{"level":"info","ts":"2026-09-18T09:51:25.229Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"e8f6a155-bde4-4be0-96ae-3fd0e3acbd3c","name":"kuberay-gateway-access","namespace":"test-ns-chvpt"}
{"level":"info","ts":"2026-09-18T09:51:25.238Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"43fe5ddf-ab1f-4654-952f-d3a5ad91cee5","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:25.238Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"43fe5ddf-ab1f-4654-952f-d3a5ad91cee5","RayCluster":{"name":"counter-bstjf","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:25.238Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"43fe5ddf-ab1f-4654-952f-d3a5ad91cee5","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T09:51:25.238Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"43fe5ddf-ab1f-4654-952f-d3a5ad91cee5","Kubernetes Job":""}
{"level":"info","ts":"2026-09-18T09:51:25.238Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"43fe5ddf-ab1f-4654-952f-d3a5ad91cee5","isJobDeleted":true}
{"level":"info","ts":"2026-09-18T09:51:25.238Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"43fe5ddf-ab1f-4654-952f-d3a5ad91cee5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kdp4b","rayClusterName":"counter-bstjf","jobDeploymentStatus":"Suspending","startTime":"2026-09-18T09:51:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-09-18T09:51:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:51:25.238Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"43fe5ddf-ab1f-4654-952f-d3a5ad91cee5","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
{"level":"info","ts":"2026-09-18T09:51:25.245Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"e8f6a155-bde4-4be0-96ae-3fd0e3acbd3c","name":"test-ns-chvpt-long-running-xrs5s","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T09:51:25.255Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"135b30fb-dd03-44b7-a505-03c9efa8c013","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:25.266Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"b0417a82-aa54-4d34-8b47-7f65de1a2305"}
{"level":"info","ts":"2026-09-18T09:51:25.266Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"e8f6a155-bde4-4be0-96ae-3fd0e3acbd3c","configMap":"kube-rbac-proxy-config-long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:25.274Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"e8f6a155-bde4-4be0-96ae-3fd0e3acbd3c","serviceAccount":"long-running-xrs5s-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:51:25.286Z","logger":"raycluster-resource","msg":"default","name":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:25.286Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-xrs5s","namespace":"test-ns-chvpt"}
{"level":"info","ts":"2026-09-18T09:51:25.307Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"e8f6a155-bde4-4be0-96ae-3fd0e3acbd3c","cluster":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:25.307Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"b6416382-f46a-4a90-bdcf-8533dd166738","namespacedName":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:25.307Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"b6416382-f46a-4a90-bdcf-8533dd166738"}
{"level":"info","ts":"2026-09-18T09:51:25.307Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"3421dd30-0524-40e1-91b8-877ee8561f0e","namespace":"test-ns-chvpt","clusterName":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:25.307Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"d416b3ec-6ba9-4df9-b99b-7c972417fd43"}
{"level":"info","ts":"2026-09-18T09:51:25.307Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"b6416382-f46a-4a90-bdcf-8533dd166738","namespace":"test-ns-chvpt","authClusters":0}
{"level":"info","ts":"2026-09-18T09:51:25.307Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"b6416382-f46a-4a90-bdcf-8533dd166738","namespace":"test-ns-chvpt"}
{"level":"info","ts":"2026-09-18T09:51:25.307Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"a4a84e35-79e8-466d-8674-9b11a8c30029","namespacedName":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:25.307Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"a4a84e35-79e8-466d-8674-9b11a8c30029"}
{"level":"info","ts":"2026-09-18T09:51:25.307Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"a4a84e35-79e8-466d-8674-9b11a8c30029","namespace":"test-ns-chvpt","authClusters":0}
{"level":"info","ts":"2026-09-18T09:51:25.307Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"long-running","namespace":"test-ns-chvpt"},"reconcileID":"9584214b-5f3c-4621-9b6a-bfbb9f705668"}
{"level":"info","ts":"2026-09-18T09:51:25.307Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"a4a84e35-79e8-466d-8674-9b11a8c30029","namespace":"test-ns-chvpt"}
{"level":"info","ts":"2026-09-18T09:51:25.330Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"dc58e492-e292-4fbb-a40a-101326485160","namespacedName":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:25.330Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"dc58e492-e292-4fbb-a40a-101326485160"}
{"level":"info","ts":"2026-09-18T09:51:25.330Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"dc58e492-e292-4fbb-a40a-101326485160","namespace":"test-ns-chvpt","authClusters":0}
{"level":"info","ts":"2026-09-18T09:51:25.330Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"dc58e492-e292-4fbb-a40a-101326485160","namespace":"test-ns-chvpt"}
{"level":"info","ts":"2026-09-18T09:51:25.354Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"9ba57866-d78e-4c24-b827-4b5ad976ae5a"}
{"level":"info","ts":"2026-09-18T09:51:25.377Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"c936e3bf-5596-4714-acaa-26870ecf3f7f"}
{"level":"info","ts":"2026-09-18T09:51:25.414Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"3421dd30-0524-40e1-91b8-877ee8561f0e","namespace":"test-ns-chvpt","clusterName":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:51:26.187Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a17139e0-d310-477d-ad61-2c5097d945aa","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:26.187Z","logger":"controllers.RayJob","msg":"The status is 'Suspended', but the suspend flag is false. Transition the status to 'New'.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a17139e0-d310-477d-ad61-2c5097d945aa"}
{"level":"info","ts":"2026-09-18T09:51:26.187Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a17139e0-d310-477d-ad61-2c5097d945aa","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-09-18T09:51:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"startTime":"2026-09-18T09:51:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:51:26.187Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a17139e0-d310-477d-ad61-2c5097d945aa","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspended","new JobDeploymentStatus":""}
{"level":"info","ts":"2026-09-18T09:51:26.199Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c73f941a-04a5-45f9-b9b8-8e0fe2f5aa00","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:26.199Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c73f941a-04a5-45f9-b9b8-8e0fe2f5aa00"}
{"level":"info","ts":"2026-09-18T09:51:26.199Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c73f941a-04a5-45f9-b9b8-8e0fe2f5aa00","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:51:26.199Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c73f941a-04a5-45f9-b9b8-8e0fe2f5aa00","oldRayJobStatus":{"rayJobInfo":{},"startTime":"2026-09-18T09:51:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:51:26.200Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c73f941a-04a5-45f9-b9b8-8e0fe2f5aa00","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:51:26.215Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"223ebfcc-e543-4ca3-9941-4db449fc9e20","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:26.215Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"223ebfcc-e543-4ca3-9941-4db449fc9e20","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.215Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"223ebfcc-e543-4ca3-9941-4db449fc9e20","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:26.215Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"223ebfcc-e543-4ca3-9941-4db449fc9e20","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:26.220Z","logger":"raycluster-resource","msg":"default","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.220Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-bpbrd","namespace":"test-ns-chvpt"}
{"level":"info","ts":"2026-09-18T09:51:26.228Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"223ebfcc-e543-4ca3-9941-4db449fc9e20","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:26.228Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c9a5bdaa-593b-4b8c-ae10-ed40c1eebf41"}
{"level":"info","ts":"2026-09-18T09:51:26.228Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c9a5bdaa-593b-4b8c-ae10-ed40c1eebf41"}
{"level":"info","ts":"2026-09-18T09:51:26.228Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c9a5bdaa-593b-4b8c-ae10-ed40c1eebf41"}
{"level":"info","ts":"2026-09-18T09:51:26.229Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"57d27f72-d467-4cb0-ba8f-df53a8b75b46","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.229Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"57d27f72-d467-4cb0-ba8f-df53a8b75b46","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.229Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"7859cef8-d433-4ec6-aeef-3929f028b55d","namespacedName":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:26.229Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"7859cef8-d433-4ec6-aeef-3929f028b55d","mode":"IntegratedOAuth","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.229Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"7859cef8-d433-4ec6-aeef-3929f028b55d","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.229Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"7859cef8-d433-4ec6-aeef-3929f028b55d","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.228Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"223ebfcc-e543-4ca3-9941-4db449fc9e20"}
{"level":"info","ts":"2026-09-18T09:51:26.229Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"223ebfcc-e543-4ca3-9941-4db449fc9e20","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:26.229Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"223ebfcc-e543-4ca3-9941-4db449fc9e20","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:51:26.229Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"46a1164a-85cc-448d-b50f-347190b85235","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:26.229Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"46a1164a-85cc-448d-b50f-347190b85235","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.229Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"46a1164a-85cc-448d-b50f-347190b85235","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:26.229Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"6f9aa6c2-0d6a-4f19-823b-dd68cdeebe6c","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.229Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"46a1164a-85cc-448d-b50f-347190b85235"}
{"level":"info","ts":"2026-09-18T09:51:26.229Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"46a1164a-85cc-448d-b50f-347190b85235","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:26.229Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"46a1164a-85cc-448d-b50f-347190b85235","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:51:26.266Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c9a5bdaa-593b-4b8c-ae10-ed40c1eebf41","name":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:26.266Z","logger":"raycluster-resource","msg":"default","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.266Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c9a5bdaa-593b-4b8c-ae10-ed40c1eebf41","error":"head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"info","ts":"2026-09-18T09:51:26.266Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-bpbrd","namespace":"test-ns-chvpt"}
{"level":"error","ts":"2026-09-18T09:51:26.266Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c9a5bdaa-593b-4b8c-ae10-ed40c1eebf41","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.266Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c9a5bdaa-593b-4b8c-ae10-ed40c1eebf41","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"info","ts":"2026-09-18T09:51:26.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c9a5bdaa-593b-4b8c-ae10-ed40c1eebf41","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c9a5bdaa-593b-4b8c-ae10-ed40c1eebf41","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c9a5bdaa-593b-4b8c-ae10-ed40c1eebf41","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.266Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c9a5bdaa-593b-4b8c-ae10-ed40c1eebf41","name":"counter-bpbrd","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:51:26.267Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"57d27f72-d467-4cb0-ba8f-df53a8b75b46","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.267Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"57d27f72-d467-4cb0-ba8f-df53a8b75b46","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.275Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"6f9aa6c2-0d6a-4f19-823b-dd68cdeebe6c","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.276Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"a726a367-eb60-4f3b-b1ba-2ba7e9919ae9","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.276Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"cb76498e-b6ce-461e-8d9f-00e962154c7a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:26.277Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"cb76498e-b6ce-461e-8d9f-00e962154c7a","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.277Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"cb76498e-b6ce-461e-8d9f-00e962154c7a","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:26.277Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"cb76498e-b6ce-461e-8d9f-00e962154c7a"}
{"level":"info","ts":"2026-09-18T09:51:26.277Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"cb76498e-b6ce-461e-8d9f-00e962154c7a","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:26.277Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"cb76498e-b6ce-461e-8d9f-00e962154c7a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:51:26.279Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"57d27f72-d467-4cb0-ba8f-df53a8b75b46","rayCluster":"counter-bpbrd","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T09:51:26.279Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"57d27f72-d467-4cb0-ba8f-df53a8b75b46","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.281Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c9a5bdaa-593b-4b8c-ae10-ed40c1eebf41","name":"counter-bpbrd","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-bpbrd\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-chvpt","name":"counter-bpbrd"}}
{"level":"info","ts":"2026-09-18T09:51:26.281Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c9a5bdaa-593b-4b8c-ae10-ed40c1eebf41"}
{"level":"error","ts":"2026-09-18T09:51:26.281Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c9a5bdaa-593b-4b8c-ae10-ed40c1eebf41","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.281Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e349a475-f0ba-4335-b002-8dabaa976324"}
{"level":"info","ts":"2026-09-18T09:51:26.281Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e349a475-f0ba-4335-b002-8dabaa976324"}
{"level":"info","ts":"2026-09-18T09:51:26.281Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e349a475-f0ba-4335-b002-8dabaa976324"}
{"level":"info","ts":"2026-09-18T09:51:26.281Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e349a475-f0ba-4335-b002-8dabaa976324","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:26.281Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e349a475-f0ba-4335-b002-8dabaa976324","error":"head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"error","ts":"2026-09-18T09:51:26.281Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e349a475-f0ba-4335-b002-8dabaa976324","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.281Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e349a475-f0ba-4335-b002-8dabaa976324","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"info","ts":"2026-09-18T09:51:26.281Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e349a475-f0ba-4335-b002-8dabaa976324","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.281Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e349a475-f0ba-4335-b002-8dabaa976324","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.281Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e349a475-f0ba-4335-b002-8dabaa976324","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.281Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e349a475-f0ba-4335-b002-8dabaa976324","name":"counter-bpbrd","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:51:26.283Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"7859cef8-d433-4ec6-aeef-3929f028b55d","name":"counter-bpbrd-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:51:26.288Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"57d27f72-d467-4cb0-ba8f-df53a8b75b46","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.294Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"a726a367-eb60-4f3b-b1ba-2ba7e9919ae9","name":"counter-bpbrd-head"}
{"level":"info","ts":"2026-09-18T09:51:26.294Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"7859cef8-d433-4ec6-aeef-3929f028b55d","name":"kuberay-gateway-access","namespace":"test-ns-chvpt","operation":"created"}
{"level":"info","ts":"2026-09-18T09:51:26.295Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e349a475-f0ba-4335-b002-8dabaa976324"}
{"level":"error","ts":"2026-09-18T09:51:26.295Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e349a475-f0ba-4335-b002-8dabaa976324","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.295Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"4cff5926-91cb-4f83-b849-f59d6414297d"}
{"level":"info","ts":"2026-09-18T09:51:26.295Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"4cff5926-91cb-4f83-b849-f59d6414297d"}
{"level":"info","ts":"2026-09-18T09:51:26.295Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"4cff5926-91cb-4f83-b849-f59d6414297d"}
{"level":"info","ts":"2026-09-18T09:51:26.295Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"4cff5926-91cb-4f83-b849-f59d6414297d","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:26.295Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"4cff5926-91cb-4f83-b849-f59d6414297d","error":"head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"error","ts":"2026-09-18T09:51:26.295Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"4cff5926-91cb-4f83-b849-f59d6414297d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.365Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"4cff5926-91cb-4f83-b849-f59d6414297d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"info","ts":"2026-09-18T09:51:26.296Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6b841cd2-1d44-46ae-9adb-19a1ef7f6c8b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:26.366Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6b841cd2-1d44-46ae-9adb-19a1ef7f6c8b","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"4cff5926-91cb-4f83-b849-f59d6414297d","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.366Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6b841cd2-1d44-46ae-9adb-19a1ef7f6c8b","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:26.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"4cff5926-91cb-4f83-b849-f59d6414297d","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"4cff5926-91cb-4f83-b849-f59d6414297d","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.366Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"4cff5926-91cb-4f83-b849-f59d6414297d","name":"counter-bpbrd","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:51:26.366Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6b841cd2-1d44-46ae-9adb-19a1ef7f6c8b"}
{"level":"info","ts":"2026-09-18T09:51:26.366Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6b841cd2-1d44-46ae-9adb-19a1ef7f6c8b","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:26.366Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6b841cd2-1d44-46ae-9adb-19a1ef7f6c8b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:26.366Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6b841cd2-1d44-46ae-9adb-19a1ef7f6c8b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:51:26.366Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"57d27f72-d467-4cb0-ba8f-df53a8b75b46","rayCluster":"counter-bpbrd","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:51:26.366Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"7859cef8-d433-4ec6-aeef-3929f028b55d","name":"test-ns-chvpt-counter-bpbrd","operation":"created"}
{"level":"info","ts":"2026-09-18T09:51:26.371Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"a726a367-eb60-4f3b-b1ba-2ba7e9919ae9","name":"counter-bpbrd-workers"}
{"level":"info","ts":"2026-09-18T09:51:26.371Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"a726a367-eb60-4f3b-b1ba-2ba7e9919ae9","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.371Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"1a64de99-c958-4c7e-8d4f-aea9a88d6913","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.375Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"7859cef8-d433-4ec6-aeef-3929f028b55d","name":"kube-rbac-proxy-config-counter-bpbrd","operation":"created"}
{"level":"info","ts":"2026-09-18T09:51:26.375Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"4cff5926-91cb-4f83-b849-f59d6414297d","name":"counter-bpbrd","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-bpbrd\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-chvpt","name":"counter-bpbrd"}}
{"level":"info","ts":"2026-09-18T09:51:26.375Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"4cff5926-91cb-4f83-b849-f59d6414297d"}
{"level":"error","ts":"2026-09-18T09:51:26.375Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"4cff5926-91cb-4f83-b849-f59d6414297d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.375Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e559cb72-3ad0-4ee7-a50f-67d2fc91a4c8"}
{"level":"info","ts":"2026-09-18T09:51:26.375Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e559cb72-3ad0-4ee7-a50f-67d2fc91a4c8"}
{"level":"info","ts":"2026-09-18T09:51:26.375Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e559cb72-3ad0-4ee7-a50f-67d2fc91a4c8"}
{"level":"info","ts":"2026-09-18T09:51:26.375Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e559cb72-3ad0-4ee7-a50f-67d2fc91a4c8","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:26.375Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e559cb72-3ad0-4ee7-a50f-67d2fc91a4c8","error":"head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"error","ts":"2026-09-18T09:51:26.375Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e559cb72-3ad0-4ee7-a50f-67d2fc91a4c8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.375Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e559cb72-3ad0-4ee7-a50f-67d2fc91a4c8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"info","ts":"2026-09-18T09:51:26.375Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e559cb72-3ad0-4ee7-a50f-67d2fc91a4c8","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.375Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e559cb72-3ad0-4ee7-a50f-67d2fc91a4c8","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.375Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e559cb72-3ad0-4ee7-a50f-67d2fc91a4c8","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.375Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e559cb72-3ad0-4ee7-a50f-67d2fc91a4c8"}
{"level":"error","ts":"2026-09-18T09:51:26.375Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"e559cb72-3ad0-4ee7-a50f-67d2fc91a4c8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.379Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"78b135d9-2614-4edc-9d82-1a4a5d8cde5e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:26.379Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"78b135d9-2614-4edc-9d82-1a4a5d8cde5e","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.379Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"78b135d9-2614-4edc-9d82-1a4a5d8cde5e","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:26.379Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"78b135d9-2614-4edc-9d82-1a4a5d8cde5e"}
{"level":"info","ts":"2026-09-18T09:51:26.379Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"78b135d9-2614-4edc-9d82-1a4a5d8cde5e","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:26.379Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"78b135d9-2614-4edc-9d82-1a4a5d8cde5e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:26.381Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"57d27f72-d467-4cb0-ba8f-df53a8b75b46","rayCluster":"counter-bpbrd","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:51:26.381Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"57d27f72-d467-4cb0-ba8f-df53a8b75b46"}
{"level":"info","ts":"2026-09-18T09:51:26.381Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"1b65d991-f117-4ced-b229-3664cea631a1","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.381Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"1b65d991-f117-4ced-b229-3664cea631a1"}
{"level":"error","ts":"2026-09-18T09:51:26.382Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"7859cef8-d433-4ec6-aeef-3929f028b55d","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-bpbrd\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.382Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"7859cef8-d433-4ec6-aeef-3929f028b55d"}
{"level":"error","ts":"2026-09-18T09:51:26.382Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"7859cef8-d433-4ec6-aeef-3929f028b55d","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-bpbrd\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.382Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"60605870-d94a-42f8-a82e-0cf1dd279a0d","namespacedName":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:26.382Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"60605870-d94a-42f8-a82e-0cf1dd279a0d","mode":"IntegratedOAuth","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.382Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"60605870-d94a-42f8-a82e-0cf1dd279a0d","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.385Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"1a64de99-c958-4c7e-8d4f-aea9a88d6913","name":"counter-bpbrd-head"}
{"level":"info","ts":"2026-09-18T09:51:26.466Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c0683db6-b92b-428c-8046-79c6962926fc"}
{"level":"info","ts":"2026-09-18T09:51:26.466Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c0683db6-b92b-428c-8046-79c6962926fc"}
{"level":"info","ts":"2026-09-18T09:51:26.466Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c0683db6-b92b-428c-8046-79c6962926fc"}
{"level":"info","ts":"2026-09-18T09:51:26.466Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c0683db6-b92b-428c-8046-79c6962926fc","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:26.466Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c0683db6-b92b-428c-8046-79c6962926fc","error":"head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"error","ts":"2026-09-18T09:51:26.466Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c0683db6-b92b-428c-8046-79c6962926fc","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.466Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c0683db6-b92b-428c-8046-79c6962926fc","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"info","ts":"2026-09-18T09:51:26.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c0683db6-b92b-428c-8046-79c6962926fc","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c0683db6-b92b-428c-8046-79c6962926fc","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c0683db6-b92b-428c-8046-79c6962926fc","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.466Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c0683db6-b92b-428c-8046-79c6962926fc"}
{"level":"error","ts":"2026-09-18T09:51:26.466Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c0683db6-b92b-428c-8046-79c6962926fc","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.467Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"60605870-d94a-42f8-a82e-0cf1dd279a0d","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.467Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e3de2992-9dc6-4970-b8d7-c1a143d0ba29","namespacedName":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:26.467Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e3de2992-9dc6-4970-b8d7-c1a143d0ba29","mode":"IntegratedOAuth","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.467Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e3de2992-9dc6-4970-b8d7-c1a143d0ba29","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.467Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"01e00693-670f-4621-8ce4-ba7a79d3ff8a","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.467Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"1227f24b-42b7-4bfb-94bd-852876d90b8d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:26.467Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"1227f24b-42b7-4bfb-94bd-852876d90b8d","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.467Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"01e00693-670f-4621-8ce4-ba7a79d3ff8a"}
{"level":"info","ts":"2026-09-18T09:51:26.467Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"1227f24b-42b7-4bfb-94bd-852876d90b8d","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:26.467Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"1227f24b-42b7-4bfb-94bd-852876d90b8d"}
{"level":"info","ts":"2026-09-18T09:51:26.467Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"1227f24b-42b7-4bfb-94bd-852876d90b8d","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:26.467Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"1227f24b-42b7-4bfb-94bd-852876d90b8d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:26.468Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"1227f24b-42b7-4bfb-94bd-852876d90b8d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:51:26.471Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"1a64de99-c958-4c7e-8d4f-aea9a88d6913","name":"counter-bpbrd-workers"}
{"level":"info","ts":"2026-09-18T09:51:26.471Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"1a64de99-c958-4c7e-8d4f-aea9a88d6913","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.471Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e930077a-66f7-48d4-8d7c-461bb2a61eff","cluster":"counter-bpbrd"}
{"level":"error","ts":"2026-09-18T09:51:26.476Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e3de2992-9dc6-4970-b8d7-c1a143d0ba29","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-bpbrd\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.476Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e3de2992-9dc6-4970-b8d7-c1a143d0ba29"}
{"level":"error","ts":"2026-09-18T09:51:26.476Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e3de2992-9dc6-4970-b8d7-c1a143d0ba29","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-bpbrd\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.481Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"6acc0717-26f2-44c2-84c8-08bea8b1d1fe","namespacedName":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:26.481Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"6acc0717-26f2-44c2-84c8-08bea8b1d1fe","mode":"IntegratedOAuth","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.481Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"6acc0717-26f2-44c2-84c8-08bea8b1d1fe","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.491Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e930077a-66f7-48d4-8d7c-461bb2a61eff","name":"counter-bpbrd-head"}
{"level":"info","ts":"2026-09-18T09:51:26.491Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f36f3bf3-fd7a-45ba-84c7-0916903c7087","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:26.491Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f36f3bf3-fd7a-45ba-84c7-0916903c7087","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.491Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f36f3bf3-fd7a-45ba-84c7-0916903c7087","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:26.491Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f36f3bf3-fd7a-45ba-84c7-0916903c7087"}
{"level":"info","ts":"2026-09-18T09:51:26.491Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f36f3bf3-fd7a-45ba-84c7-0916903c7087","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:26.491Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f36f3bf3-fd7a-45ba-84c7-0916903c7087","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:26.508Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"6acc0717-26f2-44c2-84c8-08bea8b1d1fe","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.566Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9408908e-a0da-4349-bb5b-8ba0f233a383"}
{"level":"info","ts":"2026-09-18T09:51:26.566Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9408908e-a0da-4349-bb5b-8ba0f233a383"}
{"level":"info","ts":"2026-09-18T09:51:26.566Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9408908e-a0da-4349-bb5b-8ba0f233a383"}
{"level":"info","ts":"2026-09-18T09:51:26.566Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9408908e-a0da-4349-bb5b-8ba0f233a383","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:26.566Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9408908e-a0da-4349-bb5b-8ba0f233a383","error":"head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"error","ts":"2026-09-18T09:51:26.566Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9408908e-a0da-4349-bb5b-8ba0f233a383","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.566Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9408908e-a0da-4349-bb5b-8ba0f233a383","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"info","ts":"2026-09-18T09:51:26.566Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9408908e-a0da-4349-bb5b-8ba0f233a383","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.566Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9408908e-a0da-4349-bb5b-8ba0f233a383","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.566Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9408908e-a0da-4349-bb5b-8ba0f233a383","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.566Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9408908e-a0da-4349-bb5b-8ba0f233a383"}
{"level":"error","ts":"2026-09-18T09:51:26.566Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9408908e-a0da-4349-bb5b-8ba0f233a383","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.578Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e930077a-66f7-48d4-8d7c-461bb2a61eff","name":"counter-bpbrd-workers"}
{"level":"info","ts":"2026-09-18T09:51:26.578Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e930077a-66f7-48d4-8d7c-461bb2a61eff","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:26.727Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"3fb2ce7c-a01f-47ca-b0c6-4eec906771ee"}
{"level":"info","ts":"2026-09-18T09:51:26.727Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"3fb2ce7c-a01f-47ca-b0c6-4eec906771ee"}
{"level":"info","ts":"2026-09-18T09:51:26.727Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"3fb2ce7c-a01f-47ca-b0c6-4eec906771ee"}
{"level":"info","ts":"2026-09-18T09:51:26.727Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"3fb2ce7c-a01f-47ca-b0c6-4eec906771ee","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:26.727Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"3fb2ce7c-a01f-47ca-b0c6-4eec906771ee","error":"head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"error","ts":"2026-09-18T09:51:26.727Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"3fb2ce7c-a01f-47ca-b0c6-4eec906771ee","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:26.727Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"3fb2ce7c-a01f-47ca-b0c6-4eec906771ee","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"info","ts":"2026-09-18T09:51:26.727Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"3fb2ce7c-a01f-47ca-b0c6-4eec906771ee","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.727Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"3fb2ce7c-a01f-47ca-b0c6-4eec906771ee","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.727Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"3fb2ce7c-a01f-47ca-b0c6-4eec906771ee","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:26.728Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"3fb2ce7c-a01f-47ca-b0c6-4eec906771ee"}
{"level":"error","ts":"2026-09-18T09:51:26.728Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"3fb2ce7c-a01f-47ca-b0c6-4eec906771ee","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:27.048Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"cb2cb1f6-1ff6-4755-b127-4ea7bbcb3735"}
{"level":"info","ts":"2026-09-18T09:51:27.048Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"cb2cb1f6-1ff6-4755-b127-4ea7bbcb3735"}
{"level":"info","ts":"2026-09-18T09:51:27.048Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"cb2cb1f6-1ff6-4755-b127-4ea7bbcb3735"}
{"level":"info","ts":"2026-09-18T09:51:27.048Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"cb2cb1f6-1ff6-4755-b127-4ea7bbcb3735","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:27.048Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"cb2cb1f6-1ff6-4755-b127-4ea7bbcb3735","error":"head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"error","ts":"2026-09-18T09:51:27.048Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"cb2cb1f6-1ff6-4755-b127-4ea7bbcb3735","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:27.048Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"cb2cb1f6-1ff6-4755-b127-4ea7bbcb3735","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"info","ts":"2026-09-18T09:51:27.048Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"cb2cb1f6-1ff6-4755-b127-4ea7bbcb3735","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:27.048Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"cb2cb1f6-1ff6-4755-b127-4ea7bbcb3735","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:27.048Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"cb2cb1f6-1ff6-4755-b127-4ea7bbcb3735","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:27.048Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"cb2cb1f6-1ff6-4755-b127-4ea7bbcb3735"}
{"level":"error","ts":"2026-09-18T09:51:27.048Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"cb2cb1f6-1ff6-4755-b127-4ea7bbcb3735","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:27.690Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a90d6873-73f4-46c6-a2b5-e60327eb43ea"}
{"level":"info","ts":"2026-09-18T09:51:27.690Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a90d6873-73f4-46c6-a2b5-e60327eb43ea"}
{"level":"info","ts":"2026-09-18T09:51:27.690Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a90d6873-73f4-46c6-a2b5-e60327eb43ea"}
{"level":"info","ts":"2026-09-18T09:51:27.690Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a90d6873-73f4-46c6-a2b5-e60327eb43ea","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:27.690Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a90d6873-73f4-46c6-a2b5-e60327eb43ea","error":"head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"error","ts":"2026-09-18T09:51:27.690Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a90d6873-73f4-46c6-a2b5-e60327eb43ea","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:27.690Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a90d6873-73f4-46c6-a2b5-e60327eb43ea","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found"}
{"level":"info","ts":"2026-09-18T09:51:27.690Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a90d6873-73f4-46c6-a2b5-e60327eb43ea","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:27.690Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a90d6873-73f4-46c6-a2b5-e60327eb43ea","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:27.690Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a90d6873-73f4-46c6-a2b5-e60327eb43ea","filter labels":["test-ns-chvpt",{"ray.io/cluster":"counter-bpbrd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:51:27.690Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a90d6873-73f4-46c6-a2b5-e60327eb43ea"}
{"level":"error","ts":"2026-09-18T09:51:27.690Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a90d6873-73f4-46c6-a2b5-e60327eb43ea","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-bpbrd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:28.238Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"23ec3c91-df85-4bcc-b563-65c4e4535e0c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:28.238Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"23ec3c91-df85-4bcc-b563-65c4e4535e0c","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:28.238Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"23ec3c91-df85-4bcc-b563-65c4e4535e0c","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:28.239Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"23ec3c91-df85-4bcc-b563-65c4e4535e0c"}
{"level":"info","ts":"2026-09-18T09:51:28.239Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"23ec3c91-df85-4bcc-b563-65c4e4535e0c","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:28.239Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"23ec3c91-df85-4bcc-b563-65c4e4535e0c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:28.971Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84"}
{"level":"info","ts":"2026-09-18T09:51:28.971Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84"}
{"level":"info","ts":"2026-09-18T09:51:28.971Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84"}
{"level":"info","ts":"2026-09-18T09:51:28.971Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:28.971Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84"}
{"level":"info","ts":"2026-09-18T09:51:28.971Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:51:28.971Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84"}
{"level":"info","ts":"2026-09-18T09:51:28.971Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","mode":"IntegratedOAuth","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:28.971Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","cluster":"counter-bpbrd","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:51:28.971Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","cluster":"counter-bpbrd","authType":"IntegratedOAuth","serviceAccount":"counter-bpbrd-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:51:28.971Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-bpbrd","ray.io/group":"headgroup","ray.io/identifier":"counter-bpbrd-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:51:28.971Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:51:28.971Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:51:28.971Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:51:28.971Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","enabled":true}
{"level":"info","ts":"2026-09-18T09:51:28.999Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","name":"counter-bpbrd-head-vzmvg"}
{"level":"info","ts":"2026-09-18T09:51:28.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:28.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:28.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:51:28.999Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:51:28.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:28.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:51:28.999Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84"}
{"level":"info","ts":"2026-09-18T09:51:28.999Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","nodeType":"worker","rayStartParams":{"address":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:51:28.999Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","rayStartCmd":"ray start  --address=counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:51:28.999Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:51:28.999Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","enabled":true}
{"level":"info","ts":"2026-09-18T09:51:29.025Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","name":"counter-bpbrd-small-group-worker-wffvm"}
{"level":"info","ts":"2026-09-18T09:51:29.025Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.025Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.025Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"f24f7aa5-2cee-4413-a1e4-e9bff9790c84","name":"counter-bpbrd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:51:29.045Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5"}
{"level":"info","ts":"2026-09-18T09:51:29.045Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5"}
{"level":"info","ts":"2026-09-18T09:51:29.045Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5"}
{"level":"info","ts":"2026-09-18T09:51:29.045Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:29.045Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5","name":"counter-bpbrd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5c73ca70-ebc8-4866-b03e-0a60cf042582","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5c73ca70-ebc8-4866-b03e-0a60cf042582","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5c73ca70-ebc8-4866-b03e-0a60cf042582","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5c73ca70-ebc8-4866-b03e-0a60cf042582"}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5c73ca70-ebc8-4866-b03e-0a60cf042582","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5c73ca70-ebc8-4866-b03e-0a60cf042582","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5c73ca70-ebc8-4866-b03e-0a60cf042582","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"fe3fda48-9382-48fb-a74d-e299b46cdd3d","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"fe3fda48-9382-48fb-a74d-e299b46cdd3d","rayCluster":"counter-bpbrd","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:51:29.046Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"49aa4802-576a-4594-82bb-3e27dfcc5d13","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5","name":"counter-bpbrd","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-bpbrd\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-chvpt","name":"counter-bpbrd"}}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5"}
{"level":"error","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2d0543bb-adc0-4079-832d-cdf6248b02f5","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-bpbrd\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"b42ec4ac-09a3-4608-a6cf-8a24b32896ce"}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"b42ec4ac-09a3-4608-a6cf-8a24b32896ce"}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"b42ec4ac-09a3-4608-a6cf-8a24b32896ce"}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"b42ec4ac-09a3-4608-a6cf-8a24b32896ce","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"b42ec4ac-09a3-4608-a6cf-8a24b32896ce","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"b42ec4ac-09a3-4608-a6cf-8a24b32896ce","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"b42ec4ac-09a3-4608-a6cf-8a24b32896ce","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"b42ec4ac-09a3-4608-a6cf-8a24b32896ce","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"b42ec4ac-09a3-4608-a6cf-8a24b32896ce","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"b42ec4ac-09a3-4608-a6cf-8a24b32896ce","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"b42ec4ac-09a3-4608-a6cf-8a24b32896ce","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"b42ec4ac-09a3-4608-a6cf-8a24b32896ce","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"b42ec4ac-09a3-4608-a6cf-8a24b32896ce","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"b42ec4ac-09a3-4608-a6cf-8a24b32896ce","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.066Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"b42ec4ac-09a3-4608-a6cf-8a24b32896ce","name":"counter-bpbrd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:51:29.069Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"00414d67-269d-41a1-b7ea-77dcfc01ad63","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:29.069Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"00414d67-269d-41a1-b7ea-77dcfc01ad63","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:29.069Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"00414d67-269d-41a1-b7ea-77dcfc01ad63","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:29.069Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"00414d67-269d-41a1-b7ea-77dcfc01ad63"}
{"level":"info","ts":"2026-09-18T09:51:29.069Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"00414d67-269d-41a1-b7ea-77dcfc01ad63","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:29.069Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"00414d67-269d-41a1-b7ea-77dcfc01ad63","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:29.072Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"49aa4802-576a-4594-82bb-3e27dfcc5d13","name":"counter-bpbrd-head"}
{"level":"info","ts":"2026-09-18T09:51:29.082Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff"}
{"level":"info","ts":"2026-09-18T09:51:29.082Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff"}
{"level":"info","ts":"2026-09-18T09:51:29.082Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff"}
{"level":"info","ts":"2026-09-18T09:51:29.082Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:29.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:29.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:29.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:29.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:29.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:29.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:29.082Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:29.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:29.082Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.082Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.082Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff","name":"counter-bpbrd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:51:29.083Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"08395599-9066-4aea-b46d-d85f3868107c","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:29.083Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a24b002a-2261-42e9-909b-7d47b139d734","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:29.083Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a24b002a-2261-42e9-909b-7d47b139d734","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:29.083Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a24b002a-2261-42e9-909b-7d47b139d734","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:29.083Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"08395599-9066-4aea-b46d-d85f3868107c","rayCluster":"counter-bpbrd","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:51:29.083Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a24b002a-2261-42e9-909b-7d47b139d734"}
{"level":"info","ts":"2026-09-18T09:51:29.083Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a24b002a-2261-42e9-909b-7d47b139d734","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:29.083Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a24b002a-2261-42e9-909b-7d47b139d734","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:29.083Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a24b002a-2261-42e9-909b-7d47b139d734","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:51:29.089Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"49aa4802-576a-4594-82bb-3e27dfcc5d13","name":"counter-bpbrd-workers"}
{"level":"info","ts":"2026-09-18T09:51:29.089Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"49aa4802-576a-4594-82bb-3e27dfcc5d13","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:29.089Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"800741d9-090e-4496-943b-2bcbab155833","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff","name":"counter-bpbrd","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-bpbrd\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-chvpt","name":"counter-bpbrd"}}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff"}
{"level":"error","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77567081-5d5e-47f6-83e6-71cae381b6ff","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-bpbrd\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"aa8368fc-3548-4653-85b7-8bba30ad5d9c"}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"aa8368fc-3548-4653-85b7-8bba30ad5d9c"}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"aa8368fc-3548-4653-85b7-8bba30ad5d9c"}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"aa8368fc-3548-4653-85b7-8bba30ad5d9c","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"aa8368fc-3548-4653-85b7-8bba30ad5d9c","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"aa8368fc-3548-4653-85b7-8bba30ad5d9c","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"aa8368fc-3548-4653-85b7-8bba30ad5d9c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"aa8368fc-3548-4653-85b7-8bba30ad5d9c","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"aa8368fc-3548-4653-85b7-8bba30ad5d9c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"aa8368fc-3548-4653-85b7-8bba30ad5d9c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"aa8368fc-3548-4653-85b7-8bba30ad5d9c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"aa8368fc-3548-4653-85b7-8bba30ad5d9c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"aa8368fc-3548-4653-85b7-8bba30ad5d9c","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"aa8368fc-3548-4653-85b7-8bba30ad5d9c","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"aa8368fc-3548-4653-85b7-8bba30ad5d9c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:51:29.095Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"aa8368fc-3548-4653-85b7-8bba30ad5d9c","seconds":300}
{"level":"info","ts":"2026-09-18T09:51:29.099Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"800741d9-090e-4496-943b-2bcbab155833","name":"counter-bpbrd-head"}
{"level":"info","ts":"2026-09-18T09:51:29.100Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"59f88f7b-6d07-498f-9dbe-bceb7eaf26ca"}
{"level":"info","ts":"2026-09-18T09:51:29.100Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"59f88f7b-6d07-498f-9dbe-bceb7eaf26ca"}
{"level":"info","ts":"2026-09-18T09:51:29.100Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"59f88f7b-6d07-498f-9dbe-bceb7eaf26ca"}
{"level":"info","ts":"2026-09-18T09:51:29.100Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"59f88f7b-6d07-498f-9dbe-bceb7eaf26ca","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:29.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"59f88f7b-6d07-498f-9dbe-bceb7eaf26ca","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:29.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"59f88f7b-6d07-498f-9dbe-bceb7eaf26ca","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:29.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"59f88f7b-6d07-498f-9dbe-bceb7eaf26ca","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:29.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"59f88f7b-6d07-498f-9dbe-bceb7eaf26ca","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:29.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"59f88f7b-6d07-498f-9dbe-bceb7eaf26ca","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:29.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"59f88f7b-6d07-498f-9dbe-bceb7eaf26ca","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:29.101Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"59f88f7b-6d07-498f-9dbe-bceb7eaf26ca","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:29.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"59f88f7b-6d07-498f-9dbe-bceb7eaf26ca","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:29.101Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"59f88f7b-6d07-498f-9dbe-bceb7eaf26ca","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.101Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"59f88f7b-6d07-498f-9dbe-bceb7eaf26ca","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.101Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"59f88f7b-6d07-498f-9dbe-bceb7eaf26ca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:51:29.101Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"59f88f7b-6d07-498f-9dbe-bceb7eaf26ca","seconds":300}
{"level":"info","ts":"2026-09-18T09:51:29.166Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"35bf4e08-e627-4388-9162-7f3fae915485","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:29.166Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"35bf4e08-e627-4388-9162-7f3fae915485","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:29.166Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"35bf4e08-e627-4388-9162-7f3fae915485","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:29.166Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"35bf4e08-e627-4388-9162-7f3fae915485"}
{"level":"info","ts":"2026-09-18T09:51:29.166Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"35bf4e08-e627-4388-9162-7f3fae915485","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:29.166Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"35bf4e08-e627-4388-9162-7f3fae915485","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:29.171Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"800741d9-090e-4496-943b-2bcbab155833","name":"counter-bpbrd-workers"}
{"level":"info","ts":"2026-09-18T09:51:29.171Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"800741d9-090e-4496-943b-2bcbab155833","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:29.608Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9c002b85-a10e-483e-96f7-d822442063c9"}
{"level":"info","ts":"2026-09-18T09:51:29.608Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9c002b85-a10e-483e-96f7-d822442063c9"}
{"level":"info","ts":"2026-09-18T09:51:29.608Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9c002b85-a10e-483e-96f7-d822442063c9"}
{"level":"info","ts":"2026-09-18T09:51:29.608Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9c002b85-a10e-483e-96f7-d822442063c9","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:29.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9c002b85-a10e-483e-96f7-d822442063c9","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:29.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9c002b85-a10e-483e-96f7-d822442063c9","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:29.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9c002b85-a10e-483e-96f7-d822442063c9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:29.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9c002b85-a10e-483e-96f7-d822442063c9","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:29.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9c002b85-a10e-483e-96f7-d822442063c9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:29.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9c002b85-a10e-483e-96f7-d822442063c9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:29.609Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9c002b85-a10e-483e-96f7-d822442063c9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:29.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9c002b85-a10e-483e-96f7-d822442063c9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:29.609Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9c002b85-a10e-483e-96f7-d822442063c9","podName":"counter-bpbrd-small-group-worker-wffvm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.609Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9c002b85-a10e-483e-96f7-d822442063c9","podName":"counter-bpbrd-small-group-worker-wffvm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.609Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9c002b85-a10e-483e-96f7-d822442063c9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:51:29.609Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"9c002b85-a10e-483e-96f7-d822442063c9","seconds":300}
{"level":"info","ts":"2026-09-18T09:51:29.635Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"ba5f9d4f-ec96-4d9c-b5c9-45ef215bbd5f"}
{"level":"info","ts":"2026-09-18T09:51:29.635Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"ba5f9d4f-ec96-4d9c-b5c9-45ef215bbd5f"}
{"level":"info","ts":"2026-09-18T09:51:29.635Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"ba5f9d4f-ec96-4d9c-b5c9-45ef215bbd5f"}
{"level":"info","ts":"2026-09-18T09:51:29.635Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"ba5f9d4f-ec96-4d9c-b5c9-45ef215bbd5f","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:29.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"ba5f9d4f-ec96-4d9c-b5c9-45ef215bbd5f","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:29.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"ba5f9d4f-ec96-4d9c-b5c9-45ef215bbd5f","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:29.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"ba5f9d4f-ec96-4d9c-b5c9-45ef215bbd5f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:29.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"ba5f9d4f-ec96-4d9c-b5c9-45ef215bbd5f","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:29.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"ba5f9d4f-ec96-4d9c-b5c9-45ef215bbd5f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:29.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"ba5f9d4f-ec96-4d9c-b5c9-45ef215bbd5f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:29.635Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"ba5f9d4f-ec96-4d9c-b5c9-45ef215bbd5f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:29.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"ba5f9d4f-ec96-4d9c-b5c9-45ef215bbd5f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:29.635Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"ba5f9d4f-ec96-4d9c-b5c9-45ef215bbd5f","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.635Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"ba5f9d4f-ec96-4d9c-b5c9-45ef215bbd5f","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.635Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"ba5f9d4f-ec96-4d9c-b5c9-45ef215bbd5f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:51:29.635Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"ba5f9d4f-ec96-4d9c-b5c9-45ef215bbd5f","seconds":300}
{"level":"info","ts":"2026-09-18T09:51:29.886Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d935c8f6-0937-47f2-a73b-81acc88d797d"}
{"level":"info","ts":"2026-09-18T09:51:29.886Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d935c8f6-0937-47f2-a73b-81acc88d797d"}
{"level":"info","ts":"2026-09-18T09:51:29.886Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d935c8f6-0937-47f2-a73b-81acc88d797d"}
{"level":"info","ts":"2026-09-18T09:51:29.886Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d935c8f6-0937-47f2-a73b-81acc88d797d","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:29.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d935c8f6-0937-47f2-a73b-81acc88d797d","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:29.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d935c8f6-0937-47f2-a73b-81acc88d797d","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:29.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d935c8f6-0937-47f2-a73b-81acc88d797d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:29.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d935c8f6-0937-47f2-a73b-81acc88d797d","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:29.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d935c8f6-0937-47f2-a73b-81acc88d797d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:29.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d935c8f6-0937-47f2-a73b-81acc88d797d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:29.886Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d935c8f6-0937-47f2-a73b-81acc88d797d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:29.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d935c8f6-0937-47f2-a73b-81acc88d797d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:29.887Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d935c8f6-0937-47f2-a73b-81acc88d797d","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.887Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d935c8f6-0937-47f2-a73b-81acc88d797d","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:29.887Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d935c8f6-0937-47f2-a73b-81acc88d797d","name":"counter-bpbrd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:51:29.912Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8b480f5b-737f-41b1-86ac-756d08787c69","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:29.912Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8b480f5b-737f-41b1-86ac-756d08787c69","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:29.912Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"ea55e585-6bd3-4204-ba45-d284c0cd1e24","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:29.912Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"4bec1e5d-1fdb-4bc2-80f1-6dcbe0d48c94","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:29.912Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"4bec1e5d-1fdb-4bc2-80f1-6dcbe0d48c94","pod":"counter-bpbrd-head-vzmvg","ip":"10.132.0.36"}
{"level":"info","ts":"2026-09-18T09:51:29.912Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8b480f5b-737f-41b1-86ac-756d08787c69","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:29.912Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8b480f5b-737f-41b1-86ac-756d08787c69"}
{"level":"info","ts":"2026-09-18T09:51:29.912Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8b480f5b-737f-41b1-86ac-756d08787c69","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:29.912Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8b480f5b-737f-41b1-86ac-756d08787c69","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:29.912Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8b480f5b-737f-41b1-86ac-756d08787c69","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:51:29.928Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"22e0f477-7c90-4efa-acfe-2dcabdc4c8dc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:29.928Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"22e0f477-7c90-4efa-acfe-2dcabdc4c8dc","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:29.928Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"22e0f477-7c90-4efa-acfe-2dcabdc4c8dc","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:29.928Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"22e0f477-7c90-4efa-acfe-2dcabdc4c8dc"}
{"level":"info","ts":"2026-09-18T09:51:29.928Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"22e0f477-7c90-4efa-acfe-2dcabdc4c8dc","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:29.928Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"22e0f477-7c90-4efa-acfe-2dcabdc4c8dc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:29.933Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"4bec1e5d-1fdb-4bc2-80f1-6dcbe0d48c94","certificate":"ray-head-cert-counter-bpbrd","dnsNames":["counter-bpbrd-head-svc","localhost","counter-bpbrd-head-svc.test-ns-chvpt.svc","counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local"],"ipAddresses":["10.132.0.36","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:51:29.940Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"ea55e585-6bd3-4204-ba45-d284c0cd1e24","name":"counter-bpbrd-head"}
{"level":"info","ts":"2026-09-18T09:51:29.948Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"4bec1e5d-1fdb-4bc2-80f1-6dcbe0d48c94","certificate":"ray-worker-cert-counter-bpbrd","dnsNames":["counter-bpbrd-worker-svc","localhost","counter-bpbrd-small-group","counter-bpbrd-worker-svc.test-ns-chvpt.svc","counter-bpbrd-worker-svc.test-ns-chvpt.svc.cluster.local","*.counter-bpbrd-worker-svc.test-ns-chvpt.svc","*.counter-bpbrd-worker-svc.test-ns-chvpt.svc.cluster.local","*-worker-*.test-ns-chvpt.svc","*-worker-*.test-ns-chvpt.svc.cluster.local"],"ipAddresses":["10.132.0.36","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:51:29.948Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"4bec1e5d-1fdb-4bc2-80f1-6dcbe0d48c94","rayCluster":"counter-bpbrd","podIPs":["10.132.0.36"]}
{"level":"info","ts":"2026-09-18T09:51:29.956Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"ea55e585-6bd3-4204-ba45-d284c0cd1e24","name":"counter-bpbrd-workers"}
{"level":"info","ts":"2026-09-18T09:51:29.956Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"ea55e585-6bd3-4204-ba45-d284c0cd1e24","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:30.649Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d872b7a9-f2a9-4040-95ee-d1e82497d229"}
{"level":"info","ts":"2026-09-18T09:51:30.649Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d872b7a9-f2a9-4040-95ee-d1e82497d229"}
{"level":"info","ts":"2026-09-18T09:51:30.649Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d872b7a9-f2a9-4040-95ee-d1e82497d229"}
{"level":"info","ts":"2026-09-18T09:51:30.649Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d872b7a9-f2a9-4040-95ee-d1e82497d229","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:30.649Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d872b7a9-f2a9-4040-95ee-d1e82497d229","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:30.649Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d872b7a9-f2a9-4040-95ee-d1e82497d229","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:30.649Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d872b7a9-f2a9-4040-95ee-d1e82497d229","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:30.650Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d872b7a9-f2a9-4040-95ee-d1e82497d229","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:30.650Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d872b7a9-f2a9-4040-95ee-d1e82497d229","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:30.650Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d872b7a9-f2a9-4040-95ee-d1e82497d229","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:30.650Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d872b7a9-f2a9-4040-95ee-d1e82497d229","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:30.650Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d872b7a9-f2a9-4040-95ee-d1e82497d229","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:30.650Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d872b7a9-f2a9-4040-95ee-d1e82497d229","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:30.650Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d872b7a9-f2a9-4040-95ee-d1e82497d229","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:30.650Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d872b7a9-f2a9-4040-95ee-d1e82497d229","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:51:30.650Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"d872b7a9-f2a9-4040-95ee-d1e82497d229","seconds":300}
{"level":"info","ts":"2026-09-18T09:51:31.239Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4d84f118-431e-46e7-abe3-374f42e1bcd1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:31.239Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4d84f118-431e-46e7-abe3-374f42e1bcd1","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:31.239Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4d84f118-431e-46e7-abe3-374f42e1bcd1","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:31.240Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4d84f118-431e-46e7-abe3-374f42e1bcd1"}
{"level":"info","ts":"2026-09-18T09:51:31.240Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4d84f118-431e-46e7-abe3-374f42e1bcd1","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:31.240Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4d84f118-431e-46e7-abe3-374f42e1bcd1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:31.912Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2626ddb9-c58b-456f-a1b8-8f77f99d2f58"}
{"level":"info","ts":"2026-09-18T09:51:31.912Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2626ddb9-c58b-456f-a1b8-8f77f99d2f58"}
{"level":"info","ts":"2026-09-18T09:51:31.912Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2626ddb9-c58b-456f-a1b8-8f77f99d2f58"}
{"level":"info","ts":"2026-09-18T09:51:31.912Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2626ddb9-c58b-456f-a1b8-8f77f99d2f58","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:31.913Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2626ddb9-c58b-456f-a1b8-8f77f99d2f58","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:31.913Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2626ddb9-c58b-456f-a1b8-8f77f99d2f58","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:31.913Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2626ddb9-c58b-456f-a1b8-8f77f99d2f58","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:31.913Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2626ddb9-c58b-456f-a1b8-8f77f99d2f58","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:31.913Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2626ddb9-c58b-456f-a1b8-8f77f99d2f58","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:31.913Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2626ddb9-c58b-456f-a1b8-8f77f99d2f58","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:31.913Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2626ddb9-c58b-456f-a1b8-8f77f99d2f58","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:31.913Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2626ddb9-c58b-456f-a1b8-8f77f99d2f58","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:31.913Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2626ddb9-c58b-456f-a1b8-8f77f99d2f58","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:31.913Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2626ddb9-c58b-456f-a1b8-8f77f99d2f58","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:31.913Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2626ddb9-c58b-456f-a1b8-8f77f99d2f58","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:51:31.913Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"2626ddb9-c58b-456f-a1b8-8f77f99d2f58","seconds":300}
{"level":"info","ts":"2026-09-18T09:51:34.240Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5462b5da-a002-43d2-b52a-bda48b7f9e7f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:34.240Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5462b5da-a002-43d2-b52a-bda48b7f9e7f","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:34.240Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5462b5da-a002-43d2-b52a-bda48b7f9e7f","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:34.240Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5462b5da-a002-43d2-b52a-bda48b7f9e7f"}
{"level":"info","ts":"2026-09-18T09:51:34.240Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5462b5da-a002-43d2-b52a-bda48b7f9e7f","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:34.240Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5462b5da-a002-43d2-b52a-bda48b7f9e7f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:34.924Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"11dda8f6-5de9-49e0-a27f-fb8c1d0af39a"}
{"level":"info","ts":"2026-09-18T09:51:34.924Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"11dda8f6-5de9-49e0-a27f-fb8c1d0af39a"}
{"level":"info","ts":"2026-09-18T09:51:34.924Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"11dda8f6-5de9-49e0-a27f-fb8c1d0af39a"}
{"level":"info","ts":"2026-09-18T09:51:34.924Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"11dda8f6-5de9-49e0-a27f-fb8c1d0af39a","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:34.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"11dda8f6-5de9-49e0-a27f-fb8c1d0af39a","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:34.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"11dda8f6-5de9-49e0-a27f-fb8c1d0af39a","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:34.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"11dda8f6-5de9-49e0-a27f-fb8c1d0af39a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:34.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"11dda8f6-5de9-49e0-a27f-fb8c1d0af39a","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:34.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"11dda8f6-5de9-49e0-a27f-fb8c1d0af39a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:34.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"11dda8f6-5de9-49e0-a27f-fb8c1d0af39a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:34.924Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"11dda8f6-5de9-49e0-a27f-fb8c1d0af39a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:34.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"11dda8f6-5de9-49e0-a27f-fb8c1d0af39a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:34.924Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"11dda8f6-5de9-49e0-a27f-fb8c1d0af39a","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:34.924Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"11dda8f6-5de9-49e0-a27f-fb8c1d0af39a","podName":"counter-bpbrd-head-vzmvg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:34.924Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"11dda8f6-5de9-49e0-a27f-fb8c1d0af39a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:51:34.924Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"11dda8f6-5de9-49e0-a27f-fb8c1d0af39a","seconds":300}
{"level":"info","ts":"2026-09-18T09:51:35.921Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77da1dc0-76ea-41f0-8b2f-d3f6c3038069"}
{"level":"info","ts":"2026-09-18T09:51:35.921Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77da1dc0-76ea-41f0-8b2f-d3f6c3038069"}
{"level":"info","ts":"2026-09-18T09:51:35.921Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77da1dc0-76ea-41f0-8b2f-d3f6c3038069"}
{"level":"info","ts":"2026-09-18T09:51:35.921Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77da1dc0-76ea-41f0-8b2f-d3f6c3038069","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:35.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77da1dc0-76ea-41f0-8b2f-d3f6c3038069","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:35.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77da1dc0-76ea-41f0-8b2f-d3f6c3038069","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:35.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77da1dc0-76ea-41f0-8b2f-d3f6c3038069","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:35.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77da1dc0-76ea-41f0-8b2f-d3f6c3038069","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:35.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77da1dc0-76ea-41f0-8b2f-d3f6c3038069","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:35.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77da1dc0-76ea-41f0-8b2f-d3f6c3038069","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:35.922Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77da1dc0-76ea-41f0-8b2f-d3f6c3038069","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:35.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77da1dc0-76ea-41f0-8b2f-d3f6c3038069","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:35.922Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77da1dc0-76ea-41f0-8b2f-d3f6c3038069","podName":"counter-bpbrd-head-vzmvg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:51:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:51:35.922Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77da1dc0-76ea-41f0-8b2f-d3f6c3038069","podName":"counter-bpbrd-head-vzmvg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:51:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:51:35.922Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"77da1dc0-76ea-41f0-8b2f-d3f6c3038069","name":"counter-bpbrd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:51:35.946Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"24b572c3-5741-4d75-9c86-ee249dfd992d","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:35.946Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5b73da86-8395-41e2-a19c-277a560a3704","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:35.946Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5b73da86-8395-41e2-a19c-277a560a3704","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:35.946Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5b73da86-8395-41e2-a19c-277a560a3704","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:35.946Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"eb6c1023-e81a-4b7c-9a63-f2e0e4beb2c3","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:35.947Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5b73da86-8395-41e2-a19c-277a560a3704"}
{"level":"info","ts":"2026-09-18T09:51:35.947Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5b73da86-8395-41e2-a19c-277a560a3704","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:35.947Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"eb6c1023-e81a-4b7c-9a63-f2e0e4beb2c3","pod":"counter-bpbrd-head-vzmvg","ip":"10.132.0.36"}
{"level":"info","ts":"2026-09-18T09:51:35.947Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"eb6c1023-e81a-4b7c-9a63-f2e0e4beb2c3","pod":"counter-bpbrd-small-group-worker-wffvm","ip":"10.133.0.36"}
{"level":"info","ts":"2026-09-18T09:51:35.947Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5b73da86-8395-41e2-a19c-277a560a3704","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:35.947Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5b73da86-8395-41e2-a19c-277a560a3704","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:51:35.968Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"24b572c3-5741-4d75-9c86-ee249dfd992d","name":"counter-bpbrd-head"}
{"level":"info","ts":"2026-09-18T09:51:35.978Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"eb6c1023-e81a-4b7c-9a63-f2e0e4beb2c3","certificate":"ray-head-cert-counter-bpbrd","dnsNames":["counter-bpbrd-head-svc","localhost","counter-bpbrd-head-svc.test-ns-chvpt.svc","counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local"],"ipAddresses":["10.132.0.36","10.133.0.36","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:51:35.983Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"24b572c3-5741-4d75-9c86-ee249dfd992d","name":"counter-bpbrd-workers"}
{"level":"info","ts":"2026-09-18T09:51:35.983Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"24b572c3-5741-4d75-9c86-ee249dfd992d","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:35.983Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5e4de1c0-2182-4146-bc42-e6a26dd8c133","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:35.983Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5e4de1c0-2182-4146-bc42-e6a26dd8c133","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:35.984Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5e4de1c0-2182-4146-bc42-e6a26dd8c133","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:35.984Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5e4de1c0-2182-4146-bc42-e6a26dd8c133"}
{"level":"info","ts":"2026-09-18T09:51:35.984Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5e4de1c0-2182-4146-bc42-e6a26dd8c133","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:35.984Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"5e4de1c0-2182-4146-bc42-e6a26dd8c133","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:36.002Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"eb6c1023-e81a-4b7c-9a63-f2e0e4beb2c3","certificate":"ray-worker-cert-counter-bpbrd","dnsNames":["counter-bpbrd-worker-svc","localhost","counter-bpbrd-small-group","counter-bpbrd-worker-svc.test-ns-chvpt.svc","counter-bpbrd-worker-svc.test-ns-chvpt.svc.cluster.local","*.counter-bpbrd-worker-svc.test-ns-chvpt.svc","*.counter-bpbrd-worker-svc.test-ns-chvpt.svc.cluster.local","*-worker-*.test-ns-chvpt.svc","*-worker-*.test-ns-chvpt.svc.cluster.local"],"ipAddresses":["10.132.0.36","10.133.0.36","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:51:36.003Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"eb6c1023-e81a-4b7c-9a63-f2e0e4beb2c3","rayCluster":"counter-bpbrd","podIPs":["10.132.0.36","10.133.0.36"]}
{"level":"info","ts":"2026-09-18T09:51:37.241Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d5460826-666e-4df3-8a3b-aba7ea8a5dc2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:37.241Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d5460826-666e-4df3-8a3b-aba7ea8a5dc2","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:37.241Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d5460826-666e-4df3-8a3b-aba7ea8a5dc2","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:37.241Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d5460826-666e-4df3-8a3b-aba7ea8a5dc2"}
{"level":"info","ts":"2026-09-18T09:51:37.241Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d5460826-666e-4df3-8a3b-aba7ea8a5dc2","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:37.241Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d5460826-666e-4df3-8a3b-aba7ea8a5dc2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:37.946Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"7dc56b23-260e-4f3c-aa21-cd92c8abfa0c"}
{"level":"info","ts":"2026-09-18T09:51:37.946Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"7dc56b23-260e-4f3c-aa21-cd92c8abfa0c"}
{"level":"info","ts":"2026-09-18T09:51:37.946Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"7dc56b23-260e-4f3c-aa21-cd92c8abfa0c"}
{"level":"info","ts":"2026-09-18T09:51:37.946Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"7dc56b23-260e-4f3c-aa21-cd92c8abfa0c","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:37.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"7dc56b23-260e-4f3c-aa21-cd92c8abfa0c","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:37.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"7dc56b23-260e-4f3c-aa21-cd92c8abfa0c","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:37.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"7dc56b23-260e-4f3c-aa21-cd92c8abfa0c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:37.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"7dc56b23-260e-4f3c-aa21-cd92c8abfa0c","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:37.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"7dc56b23-260e-4f3c-aa21-cd92c8abfa0c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:37.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"7dc56b23-260e-4f3c-aa21-cd92c8abfa0c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:37.947Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"7dc56b23-260e-4f3c-aa21-cd92c8abfa0c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:37.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"7dc56b23-260e-4f3c-aa21-cd92c8abfa0c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:37.947Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"7dc56b23-260e-4f3c-aa21-cd92c8abfa0c","podName":"counter-bpbrd-head-vzmvg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:51:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:51:37.947Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"7dc56b23-260e-4f3c-aa21-cd92c8abfa0c","podName":"counter-bpbrd-head-vzmvg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:51:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:51:37.947Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"7dc56b23-260e-4f3c-aa21-cd92c8abfa0c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:51:37.947Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"7dc56b23-260e-4f3c-aa21-cd92c8abfa0c","seconds":300}
{"level":"info","ts":"2026-09-18T09:51:40.242Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"bc6f8db1-2bbc-4f7b-ab1b-f32f0a52cd07","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:40.242Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"bc6f8db1-2bbc-4f7b-ab1b-f32f0a52cd07","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:40.242Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"bc6f8db1-2bbc-4f7b-ab1b-f32f0a52cd07","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:40.242Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"bc6f8db1-2bbc-4f7b-ab1b-f32f0a52cd07"}
{"level":"info","ts":"2026-09-18T09:51:40.242Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"bc6f8db1-2bbc-4f7b-ab1b-f32f0a52cd07","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:40.242Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"bc6f8db1-2bbc-4f7b-ab1b-f32f0a52cd07","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:43.242Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d6f0e600-ffb3-4eed-b6d7-8ea2f4411709","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:43.242Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d6f0e600-ffb3-4eed-b6d7-8ea2f4411709","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:43.242Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d6f0e600-ffb3-4eed-b6d7-8ea2f4411709","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:43.242Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d6f0e600-ffb3-4eed-b6d7-8ea2f4411709"}
{"level":"info","ts":"2026-09-18T09:51:43.242Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d6f0e600-ffb3-4eed-b6d7-8ea2f4411709","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:43.242Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d6f0e600-ffb3-4eed-b6d7-8ea2f4411709","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:46.243Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2caf9e21-351d-4f73-818f-2590db3f7ec8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:46.243Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2caf9e21-351d-4f73-818f-2590db3f7ec8","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:46.243Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2caf9e21-351d-4f73-818f-2590db3f7ec8","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:46.243Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2caf9e21-351d-4f73-818f-2590db3f7ec8"}
{"level":"info","ts":"2026-09-18T09:51:46.243Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2caf9e21-351d-4f73-818f-2590db3f7ec8","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:46.243Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2caf9e21-351d-4f73-818f-2590db3f7ec8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:46.954Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"610273f0-74eb-41e2-83a8-9875f2283023"}
{"level":"info","ts":"2026-09-18T09:51:46.954Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"610273f0-74eb-41e2-83a8-9875f2283023"}
{"level":"info","ts":"2026-09-18T09:51:46.954Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"610273f0-74eb-41e2-83a8-9875f2283023"}
{"level":"info","ts":"2026-09-18T09:51:46.954Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"610273f0-74eb-41e2-83a8-9875f2283023","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:46.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"610273f0-74eb-41e2-83a8-9875f2283023","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:46.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"610273f0-74eb-41e2-83a8-9875f2283023","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:46.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"610273f0-74eb-41e2-83a8-9875f2283023","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:46.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"610273f0-74eb-41e2-83a8-9875f2283023","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:46.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"610273f0-74eb-41e2-83a8-9875f2283023","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:46.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"610273f0-74eb-41e2-83a8-9875f2283023","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:46.954Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"610273f0-74eb-41e2-83a8-9875f2283023","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:46.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"610273f0-74eb-41e2-83a8-9875f2283023","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:46.954Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"610273f0-74eb-41e2-83a8-9875f2283023","podName":"counter-bpbrd-small-group-worker-wffvm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:46.954Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"610273f0-74eb-41e2-83a8-9875f2283023","podName":"counter-bpbrd-small-group-worker-wffvm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:46.954Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"610273f0-74eb-41e2-83a8-9875f2283023","name":"counter-bpbrd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:51:46.969Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"188e1ceb-eb3e-4f6a-94a4-d101acc0169b","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:46.969Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"38dd6c77-d828-4ba7-932b-14b2e335bad0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:46.969Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"38dd6c77-d828-4ba7-932b-14b2e335bad0","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:46.969Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"38dd6c77-d828-4ba7-932b-14b2e335bad0","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:46.969Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"53a2e90a-bff4-416f-adbb-ad22948b69a8","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:46.969Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"53a2e90a-bff4-416f-adbb-ad22948b69a8","pod":"counter-bpbrd-head-vzmvg","ip":"10.132.0.36"}
{"level":"info","ts":"2026-09-18T09:51:46.969Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"53a2e90a-bff4-416f-adbb-ad22948b69a8","pod":"counter-bpbrd-small-group-worker-wffvm","ip":"10.133.0.36"}
{"level":"info","ts":"2026-09-18T09:51:46.969Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"38dd6c77-d828-4ba7-932b-14b2e335bad0"}
{"level":"info","ts":"2026-09-18T09:51:46.969Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"38dd6c77-d828-4ba7-932b-14b2e335bad0","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:46.969Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"38dd6c77-d828-4ba7-932b-14b2e335bad0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:46.969Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"38dd6c77-d828-4ba7-932b-14b2e335bad0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:51:46.969Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"53a2e90a-bff4-416f-adbb-ad22948b69a8","rayCluster":"counter-bpbrd","podIPs":["10.132.0.36","10.133.0.36"]}
{"level":"info","ts":"2026-09-18T09:51:46.984Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"912c3108-c973-4661-bd5d-c61344c39235","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:46.984Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"912c3108-c973-4661-bd5d-c61344c39235","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:46.984Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"912c3108-c973-4661-bd5d-c61344c39235","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:46.984Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"912c3108-c973-4661-bd5d-c61344c39235"}
{"level":"info","ts":"2026-09-18T09:51:46.984Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"912c3108-c973-4661-bd5d-c61344c39235","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:46.984Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"912c3108-c973-4661-bd5d-c61344c39235","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:46.987Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"188e1ceb-eb3e-4f6a-94a4-d101acc0169b","name":"counter-bpbrd-head"}
{"level":"info","ts":"2026-09-18T09:51:47.000Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"188e1ceb-eb3e-4f6a-94a4-d101acc0169b","name":"counter-bpbrd-workers"}
{"level":"info","ts":"2026-09-18T09:51:47.000Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"188e1ceb-eb3e-4f6a-94a4-d101acc0169b","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:48.968Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8d5c6f75-0483-433e-bc01-42509085b47c"}
{"level":"info","ts":"2026-09-18T09:51:48.968Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8d5c6f75-0483-433e-bc01-42509085b47c"}
{"level":"info","ts":"2026-09-18T09:51:48.968Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8d5c6f75-0483-433e-bc01-42509085b47c"}
{"level":"info","ts":"2026-09-18T09:51:48.968Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8d5c6f75-0483-433e-bc01-42509085b47c","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:51:48.968Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8d5c6f75-0483-433e-bc01-42509085b47c","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:51:48.968Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8d5c6f75-0483-433e-bc01-42509085b47c","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:48.968Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8d5c6f75-0483-433e-bc01-42509085b47c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:51:48.968Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8d5c6f75-0483-433e-bc01-42509085b47c","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:51:48.968Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8d5c6f75-0483-433e-bc01-42509085b47c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:48.968Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8d5c6f75-0483-433e-bc01-42509085b47c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:51:48.969Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8d5c6f75-0483-433e-bc01-42509085b47c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:51:48.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8d5c6f75-0483-433e-bc01-42509085b47c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:51:48.969Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8d5c6f75-0483-433e-bc01-42509085b47c","podName":"counter-bpbrd-small-group-worker-wffvm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:48.969Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8d5c6f75-0483-433e-bc01-42509085b47c","podName":"counter-bpbrd-small-group-worker-wffvm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:51:48.969Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8d5c6f75-0483-433e-bc01-42509085b47c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:51:48.969Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8d5c6f75-0483-433e-bc01-42509085b47c","seconds":300}
{"level":"info","ts":"2026-09-18T09:51:49.244Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c83b6e9e-30f5-4cef-b250-9ea9a5ec7e5b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:49.245Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c83b6e9e-30f5-4cef-b250-9ea9a5ec7e5b","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:49.245Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c83b6e9e-30f5-4cef-b250-9ea9a5ec7e5b","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:49.245Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c83b6e9e-30f5-4cef-b250-9ea9a5ec7e5b"}
{"level":"info","ts":"2026-09-18T09:51:49.245Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c83b6e9e-30f5-4cef-b250-9ea9a5ec7e5b","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:49.245Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c83b6e9e-30f5-4cef-b250-9ea9a5ec7e5b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:52.245Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"76963bf8-df21-4bfe-bc9d-8c2d2ad109f5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:52.245Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"76963bf8-df21-4bfe-bc9d-8c2d2ad109f5","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:52.246Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"76963bf8-df21-4bfe-bc9d-8c2d2ad109f5","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:52.246Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"76963bf8-df21-4bfe-bc9d-8c2d2ad109f5"}
{"level":"info","ts":"2026-09-18T09:51:52.246Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"76963bf8-df21-4bfe-bc9d-8c2d2ad109f5","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:52.246Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"76963bf8-df21-4bfe-bc9d-8c2d2ad109f5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:55.246Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"02050509-4552-47a2-9772-feb1b4208e06","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:55.246Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"02050509-4552-47a2-9772-feb1b4208e06","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:55.246Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"02050509-4552-47a2-9772-feb1b4208e06","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:55.246Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"02050509-4552-47a2-9772-feb1b4208e06"}
{"level":"info","ts":"2026-09-18T09:51:55.246Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"02050509-4552-47a2-9772-feb1b4208e06","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:55.246Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"02050509-4552-47a2-9772-feb1b4208e06","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:51:56.382Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"fac55f8f-9222-48f9-b131-791c8a7f4352","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:56.382Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"fac55f8f-9222-48f9-b131-791c8a7f4352","pod":"counter-bpbrd-head-vzmvg","ip":"10.132.0.36"}
{"level":"info","ts":"2026-09-18T09:51:56.382Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"fac55f8f-9222-48f9-b131-791c8a7f4352","pod":"counter-bpbrd-small-group-worker-wffvm","ip":"10.133.0.36"}
{"level":"info","ts":"2026-09-18T09:51:56.382Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"fac55f8f-9222-48f9-b131-791c8a7f4352","rayCluster":"counter-bpbrd","podIPs":["10.132.0.36","10.133.0.36"]}
{"level":"info","ts":"2026-09-18T09:51:58.247Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"3f35db6a-0a41-42c0-8c24-b58216252490","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:51:58.247Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"3f35db6a-0a41-42c0-8c24-b58216252490","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:51:58.247Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"3f35db6a-0a41-42c0-8c24-b58216252490","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:51:58.247Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"3f35db6a-0a41-42c0-8c24-b58216252490"}
{"level":"info","ts":"2026-09-18T09:51:58.247Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"3f35db6a-0a41-42c0-8c24-b58216252490","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:51:58.247Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"3f35db6a-0a41-42c0-8c24-b58216252490","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:01.248Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"40abbab5-6cf1-4311-8db0-55ada61605a8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:01.248Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"40abbab5-6cf1-4311-8db0-55ada61605a8","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:01.248Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"40abbab5-6cf1-4311-8db0-55ada61605a8","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:01.249Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"40abbab5-6cf1-4311-8db0-55ada61605a8"}
{"level":"info","ts":"2026-09-18T09:52:01.249Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"40abbab5-6cf1-4311-8db0-55ada61605a8","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:01.249Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"40abbab5-6cf1-4311-8db0-55ada61605a8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:04.249Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4d474907-23ef-4eff-8ab4-0c277cc700cf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:04.249Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4d474907-23ef-4eff-8ab4-0c277cc700cf","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:04.249Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4d474907-23ef-4eff-8ab4-0c277cc700cf","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:04.249Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4d474907-23ef-4eff-8ab4-0c277cc700cf"}
{"level":"info","ts":"2026-09-18T09:52:04.249Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4d474907-23ef-4eff-8ab4-0c277cc700cf","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:04.249Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4d474907-23ef-4eff-8ab4-0c277cc700cf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:07.250Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6405f190-a967-4766-96d7-e08383fb6739","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:07.250Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6405f190-a967-4766-96d7-e08383fb6739","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:07.250Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6405f190-a967-4766-96d7-e08383fb6739","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:07.250Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6405f190-a967-4766-96d7-e08383fb6739"}
{"level":"info","ts":"2026-09-18T09:52:07.250Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6405f190-a967-4766-96d7-e08383fb6739","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:07.250Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6405f190-a967-4766-96d7-e08383fb6739","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:10.251Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"3d344429-c971-4654-ba3e-f54a7de8c925","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:10.251Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"3d344429-c971-4654-ba3e-f54a7de8c925","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:10.251Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"3d344429-c971-4654-ba3e-f54a7de8c925","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:10.251Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"3d344429-c971-4654-ba3e-f54a7de8c925"}
{"level":"info","ts":"2026-09-18T09:52:10.251Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"3d344429-c971-4654-ba3e-f54a7de8c925","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:10.251Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"3d344429-c971-4654-ba3e-f54a7de8c925","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:13.252Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c3567da7-9008-4587-9974-4f638f399989","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:13.252Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c3567da7-9008-4587-9974-4f638f399989","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:13.252Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c3567da7-9008-4587-9974-4f638f399989","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:13.252Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c3567da7-9008-4587-9974-4f638f399989"}
{"level":"info","ts":"2026-09-18T09:52:13.252Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c3567da7-9008-4587-9974-4f638f399989","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:13.252Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c3567da7-9008-4587-9974-4f638f399989","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:16.253Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f073c3be-8e96-4db1-b952-6f3e53a88919","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:16.253Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f073c3be-8e96-4db1-b952-6f3e53a88919","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:16.253Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f073c3be-8e96-4db1-b952-6f3e53a88919","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:16.253Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f073c3be-8e96-4db1-b952-6f3e53a88919"}
{"level":"info","ts":"2026-09-18T09:52:16.253Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f073c3be-8e96-4db1-b952-6f3e53a88919","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:16.253Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f073c3be-8e96-4db1-b952-6f3e53a88919","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:19.254Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8405ba62-df7a-47af-9c37-c4b1af8f1dea","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:19.254Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8405ba62-df7a-47af-9c37-c4b1af8f1dea","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:19.254Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8405ba62-df7a-47af-9c37-c4b1af8f1dea","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:19.254Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8405ba62-df7a-47af-9c37-c4b1af8f1dea"}
{"level":"info","ts":"2026-09-18T09:52:19.254Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8405ba62-df7a-47af-9c37-c4b1af8f1dea","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:19.254Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8405ba62-df7a-47af-9c37-c4b1af8f1dea","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:22.254Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7d58af27-2e32-46f4-9e83-6b8a1392fb99","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:22.254Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7d58af27-2e32-46f4-9e83-6b8a1392fb99","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:22.255Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7d58af27-2e32-46f4-9e83-6b8a1392fb99","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:22.255Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7d58af27-2e32-46f4-9e83-6b8a1392fb99"}
{"level":"info","ts":"2026-09-18T09:52:22.255Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7d58af27-2e32-46f4-9e83-6b8a1392fb99","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:22.255Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7d58af27-2e32-46f4-9e83-6b8a1392fb99","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:24.088Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"3de16954-3425-4823-8fda-a098e83023cb","namespace":"test-ns-chvpt","clusterName":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:52:24.088Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-xrs5s","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"long-running-xrs5s","reconcileID":"3de16954-3425-4823-8fda-a098e83023cb","namespace":"test-ns-chvpt","clusterName":"long-running-xrs5s"}
{"level":"info","ts":"2026-09-18T09:52:25.256Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a07eaaff-1593-44f7-b01f-f972b8cf8957","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:25.256Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a07eaaff-1593-44f7-b01f-f972b8cf8957","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:25.256Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a07eaaff-1593-44f7-b01f-f972b8cf8957","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:25.256Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a07eaaff-1593-44f7-b01f-f972b8cf8957"}
{"level":"info","ts":"2026-09-18T09:52:25.256Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a07eaaff-1593-44f7-b01f-f972b8cf8957","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:25.256Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a07eaaff-1593-44f7-b01f-f972b8cf8957","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:28.257Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a1053d78-bf53-4f13-a45b-4a6d91a0ffde","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:28.257Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a1053d78-bf53-4f13-a45b-4a6d91a0ffde","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:28.257Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a1053d78-bf53-4f13-a45b-4a6d91a0ffde","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:28.257Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a1053d78-bf53-4f13-a45b-4a6d91a0ffde"}
{"level":"info","ts":"2026-09-18T09:52:28.257Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a1053d78-bf53-4f13-a45b-4a6d91a0ffde","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:28.257Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a1053d78-bf53-4f13-a45b-4a6d91a0ffde","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:31.258Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"eee8e360-b1e0-48cb-a9f4-bcce264e3164","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:31.258Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"eee8e360-b1e0-48cb-a9f4-bcce264e3164","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:31.258Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"eee8e360-b1e0-48cb-a9f4-bcce264e3164","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:31.258Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"eee8e360-b1e0-48cb-a9f4-bcce264e3164"}
{"level":"info","ts":"2026-09-18T09:52:31.258Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"eee8e360-b1e0-48cb-a9f4-bcce264e3164","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:31.258Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"eee8e360-b1e0-48cb-a9f4-bcce264e3164","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:34.259Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ff5b0a85-ba68-427d-a22d-e384d9ffab50","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:34.259Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ff5b0a85-ba68-427d-a22d-e384d9ffab50","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:34.259Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ff5b0a85-ba68-427d-a22d-e384d9ffab50","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:34.259Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ff5b0a85-ba68-427d-a22d-e384d9ffab50"}
{"level":"info","ts":"2026-09-18T09:52:34.259Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ff5b0a85-ba68-427d-a22d-e384d9ffab50","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:34.259Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ff5b0a85-ba68-427d-a22d-e384d9ffab50","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:37.260Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7a88c6ce-9013-41ea-b0e4-5a80c21512b3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:37.260Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7a88c6ce-9013-41ea-b0e4-5a80c21512b3","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:37.260Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7a88c6ce-9013-41ea-b0e4-5a80c21512b3","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:37.260Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7a88c6ce-9013-41ea-b0e4-5a80c21512b3"}
{"level":"info","ts":"2026-09-18T09:52:37.260Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7a88c6ce-9013-41ea-b0e4-5a80c21512b3","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:37.260Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7a88c6ce-9013-41ea-b0e4-5a80c21512b3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:40.261Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e212ca17-cdf2-4262-9509-51326decafa9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:40.261Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e212ca17-cdf2-4262-9509-51326decafa9","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:40.261Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e212ca17-cdf2-4262-9509-51326decafa9","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:40.261Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e212ca17-cdf2-4262-9509-51326decafa9"}
{"level":"info","ts":"2026-09-18T09:52:40.261Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e212ca17-cdf2-4262-9509-51326decafa9","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:40.261Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e212ca17-cdf2-4262-9509-51326decafa9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:40.932Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"34f1eec0-3f59-495f-be84-b04fd3f0bb9d"}
{"level":"info","ts":"2026-09-18T09:52:40.932Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"34f1eec0-3f59-495f-be84-b04fd3f0bb9d"}
{"level":"info","ts":"2026-09-18T09:52:40.932Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"34f1eec0-3f59-495f-be84-b04fd3f0bb9d"}
{"level":"info","ts":"2026-09-18T09:52:40.933Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"34f1eec0-3f59-495f-be84-b04fd3f0bb9d","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:52:40.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"34f1eec0-3f59-495f-be84-b04fd3f0bb9d","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:52:40.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"34f1eec0-3f59-495f-be84-b04fd3f0bb9d","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:52:40.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"34f1eec0-3f59-495f-be84-b04fd3f0bb9d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:52:40.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"34f1eec0-3f59-495f-be84-b04fd3f0bb9d","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:52:40.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"34f1eec0-3f59-495f-be84-b04fd3f0bb9d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:52:40.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"34f1eec0-3f59-495f-be84-b04fd3f0bb9d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:52:40.933Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"34f1eec0-3f59-495f-be84-b04fd3f0bb9d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:52:40.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"34f1eec0-3f59-495f-be84-b04fd3f0bb9d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:52:40.933Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"34f1eec0-3f59-495f-be84-b04fd3f0bb9d","podName":"counter-bpbrd-small-group-worker-wffvm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:52:40.933Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"34f1eec0-3f59-495f-be84-b04fd3f0bb9d","podName":"counter-bpbrd-small-group-worker-wffvm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:52:40.933Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"34f1eec0-3f59-495f-be84-b04fd3f0bb9d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:52:40.933Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"34f1eec0-3f59-495f-be84-b04fd3f0bb9d","seconds":300}
{"level":"info","ts":"2026-09-18T09:52:41.931Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c4eb16a2-8e2b-4b2c-9308-0370f6956dd1"}
{"level":"info","ts":"2026-09-18T09:52:41.931Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c4eb16a2-8e2b-4b2c-9308-0370f6956dd1"}
{"level":"info","ts":"2026-09-18T09:52:41.931Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c4eb16a2-8e2b-4b2c-9308-0370f6956dd1"}
{"level":"info","ts":"2026-09-18T09:52:41.931Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c4eb16a2-8e2b-4b2c-9308-0370f6956dd1","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:52:41.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c4eb16a2-8e2b-4b2c-9308-0370f6956dd1","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:52:41.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c4eb16a2-8e2b-4b2c-9308-0370f6956dd1","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:52:41.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c4eb16a2-8e2b-4b2c-9308-0370f6956dd1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:52:41.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c4eb16a2-8e2b-4b2c-9308-0370f6956dd1","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:52:41.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c4eb16a2-8e2b-4b2c-9308-0370f6956dd1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:52:41.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c4eb16a2-8e2b-4b2c-9308-0370f6956dd1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:52:41.931Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c4eb16a2-8e2b-4b2c-9308-0370f6956dd1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:52:41.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c4eb16a2-8e2b-4b2c-9308-0370f6956dd1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:52:41.931Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c4eb16a2-8e2b-4b2c-9308-0370f6956dd1","podName":"counter-bpbrd-small-group-worker-wffvm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:52:41.931Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c4eb16a2-8e2b-4b2c-9308-0370f6956dd1","podName":"counter-bpbrd-small-group-worker-wffvm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:52:41.931Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c4eb16a2-8e2b-4b2c-9308-0370f6956dd1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:52:41.931Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"c4eb16a2-8e2b-4b2c-9308-0370f6956dd1","seconds":300}
{"level":"info","ts":"2026-09-18T09:52:43.262Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ec08c41a-91af-4ab7-8e87-fb3dea4343f9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:43.262Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ec08c41a-91af-4ab7-8e87-fb3dea4343f9","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:43.262Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ec08c41a-91af-4ab7-8e87-fb3dea4343f9","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:43.263Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ec08c41a-91af-4ab7-8e87-fb3dea4343f9"}
{"level":"info","ts":"2026-09-18T09:52:43.263Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ec08c41a-91af-4ab7-8e87-fb3dea4343f9","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:43.263Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ec08c41a-91af-4ab7-8e87-fb3dea4343f9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:46.263Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f3f170d2-851f-4a73-966a-1df0d12fe8a6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:46.263Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f3f170d2-851f-4a73-966a-1df0d12fe8a6","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:46.263Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f3f170d2-851f-4a73-966a-1df0d12fe8a6","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:46.264Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f3f170d2-851f-4a73-966a-1df0d12fe8a6"}
{"level":"info","ts":"2026-09-18T09:52:46.264Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f3f170d2-851f-4a73-966a-1df0d12fe8a6","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:46.264Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"f3f170d2-851f-4a73-966a-1df0d12fe8a6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:49.265Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c559c2c6-b013-4c68-9da3-25f7e57c30a6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:49.265Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c559c2c6-b013-4c68-9da3-25f7e57c30a6","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:49.265Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c559c2c6-b013-4c68-9da3-25f7e57c30a6","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:49.265Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c559c2c6-b013-4c68-9da3-25f7e57c30a6"}
{"level":"info","ts":"2026-09-18T09:52:49.265Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c559c2c6-b013-4c68-9da3-25f7e57c30a6","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:49.265Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"c559c2c6-b013-4c68-9da3-25f7e57c30a6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:50.965Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a4d83080-8b40-4430-9ffe-d30a0bd26eb2"}
{"level":"info","ts":"2026-09-18T09:52:50.965Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a4d83080-8b40-4430-9ffe-d30a0bd26eb2"}
{"level":"info","ts":"2026-09-18T09:52:50.965Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a4d83080-8b40-4430-9ffe-d30a0bd26eb2"}
{"level":"info","ts":"2026-09-18T09:52:50.965Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a4d83080-8b40-4430-9ffe-d30a0bd26eb2","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:52:50.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a4d83080-8b40-4430-9ffe-d30a0bd26eb2","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:52:50.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a4d83080-8b40-4430-9ffe-d30a0bd26eb2","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:52:50.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a4d83080-8b40-4430-9ffe-d30a0bd26eb2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:52:50.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a4d83080-8b40-4430-9ffe-d30a0bd26eb2","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:52:50.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a4d83080-8b40-4430-9ffe-d30a0bd26eb2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:52:50.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a4d83080-8b40-4430-9ffe-d30a0bd26eb2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:52:50.965Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a4d83080-8b40-4430-9ffe-d30a0bd26eb2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:52:50.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a4d83080-8b40-4430-9ffe-d30a0bd26eb2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:52:50.965Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a4d83080-8b40-4430-9ffe-d30a0bd26eb2","podName":"counter-bpbrd-small-group-worker-wffvm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:52:50.965Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a4d83080-8b40-4430-9ffe-d30a0bd26eb2","podName":"counter-bpbrd-small-group-worker-wffvm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:52:50.965Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a4d83080-8b40-4430-9ffe-d30a0bd26eb2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:52:50.965Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"a4d83080-8b40-4430-9ffe-d30a0bd26eb2","seconds":300}
{"level":"info","ts":"2026-09-18T09:52:51.978Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"410b938c-db41-4b32-afb7-a2e28fef1750"}
{"level":"info","ts":"2026-09-18T09:52:51.978Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"410b938c-db41-4b32-afb7-a2e28fef1750"}
{"level":"info","ts":"2026-09-18T09:52:51.978Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"410b938c-db41-4b32-afb7-a2e28fef1750"}
{"level":"info","ts":"2026-09-18T09:52:51.978Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"410b938c-db41-4b32-afb7-a2e28fef1750","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:52:51.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"410b938c-db41-4b32-afb7-a2e28fef1750","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:52:51.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"410b938c-db41-4b32-afb7-a2e28fef1750","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:52:51.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"410b938c-db41-4b32-afb7-a2e28fef1750","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:52:51.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"410b938c-db41-4b32-afb7-a2e28fef1750","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:52:51.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"410b938c-db41-4b32-afb7-a2e28fef1750","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:52:51.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"410b938c-db41-4b32-afb7-a2e28fef1750","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:52:51.979Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"410b938c-db41-4b32-afb7-a2e28fef1750","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:52:51.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"410b938c-db41-4b32-afb7-a2e28fef1750","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:52:51.979Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"410b938c-db41-4b32-afb7-a2e28fef1750","podName":"counter-bpbrd-small-group-worker-wffvm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:51:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:52:51.979Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"410b938c-db41-4b32-afb7-a2e28fef1750","podName":"counter-bpbrd-small-group-worker-wffvm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:51:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:52:51.979Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"410b938c-db41-4b32-afb7-a2e28fef1750","name":"counter-bpbrd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:52:51.995Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e1634ed2-a49a-4851-8b16-274986201253","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:51.995Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ea68b05b-dba4-4bb6-82bf-00b8be8ccd0f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:51.995Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ea68b05b-dba4-4bb6-82bf-00b8be8ccd0f","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:51.995Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ea68b05b-dba4-4bb6-82bf-00b8be8ccd0f","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:51.995Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ea68b05b-dba4-4bb6-82bf-00b8be8ccd0f"}
{"level":"info","ts":"2026-09-18T09:52:51.995Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f023b58e-bb76-469e-8134-4e7e97b7fb36","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:51.995Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ea68b05b-dba4-4bb6-82bf-00b8be8ccd0f","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:51.995Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ea68b05b-dba4-4bb6-82bf-00b8be8ccd0f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:51:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:51.995Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"ea68b05b-dba4-4bb6-82bf-00b8be8ccd0f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:52:51.995Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f023b58e-bb76-469e-8134-4e7e97b7fb36","pod":"counter-bpbrd-head-vzmvg","ip":"10.132.0.36"}
{"level":"info","ts":"2026-09-18T09:52:51.995Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f023b58e-bb76-469e-8134-4e7e97b7fb36","pod":"counter-bpbrd-small-group-worker-wffvm","ip":"10.133.0.36"}
{"level":"info","ts":"2026-09-18T09:52:51.995Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f023b58e-bb76-469e-8134-4e7e97b7fb36","rayCluster":"counter-bpbrd","podIPs":["10.132.0.36","10.133.0.36"]}
{"level":"info","ts":"2026-09-18T09:52:52.010Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"224a53eb-fcd4-4e8c-be44-72e924d93530","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:52.010Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"224a53eb-fcd4-4e8c-be44-72e924d93530","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:52.010Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"224a53eb-fcd4-4e8c-be44-72e924d93530","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:52.010Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"224a53eb-fcd4-4e8c-be44-72e924d93530"}
{"level":"info","ts":"2026-09-18T09:52:52.010Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"224a53eb-fcd4-4e8c-be44-72e924d93530","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:52.010Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"224a53eb-fcd4-4e8c-be44-72e924d93530","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:52.015Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e1634ed2-a49a-4851-8b16-274986201253","name":"counter-bpbrd-head"}
{"level":"info","ts":"2026-09-18T09:52:52.028Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e1634ed2-a49a-4851-8b16-274986201253","name":"counter-bpbrd-workers"}
{"level":"info","ts":"2026-09-18T09:52:52.028Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e1634ed2-a49a-4851-8b16-274986201253","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:52.266Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2ae61a54-9ca8-4bc1-9064-4b6fba8b7e6a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:52.266Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2ae61a54-9ca8-4bc1-9064-4b6fba8b7e6a","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:52.266Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2ae61a54-9ca8-4bc1-9064-4b6fba8b7e6a","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:52.266Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2ae61a54-9ca8-4bc1-9064-4b6fba8b7e6a"}
{"level":"info","ts":"2026-09-18T09:52:52.266Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2ae61a54-9ca8-4bc1-9064-4b6fba8b7e6a","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:52.266Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2ae61a54-9ca8-4bc1-9064-4b6fba8b7e6a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:53.996Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8993a827-5696-462b-b15f-377e4e8759d1"}
{"level":"info","ts":"2026-09-18T09:52:53.996Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8993a827-5696-462b-b15f-377e4e8759d1"}
{"level":"info","ts":"2026-09-18T09:52:53.996Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8993a827-5696-462b-b15f-377e4e8759d1"}
{"level":"info","ts":"2026-09-18T09:52:53.996Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8993a827-5696-462b-b15f-377e4e8759d1","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:52:53.996Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8993a827-5696-462b-b15f-377e4e8759d1","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:52:53.996Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8993a827-5696-462b-b15f-377e4e8759d1","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:52:53.996Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8993a827-5696-462b-b15f-377e4e8759d1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:52:53.996Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8993a827-5696-462b-b15f-377e4e8759d1","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:52:53.996Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8993a827-5696-462b-b15f-377e4e8759d1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:52:53.996Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8993a827-5696-462b-b15f-377e4e8759d1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:52:53.996Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8993a827-5696-462b-b15f-377e4e8759d1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:52:53.996Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8993a827-5696-462b-b15f-377e4e8759d1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:52:53.996Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8993a827-5696-462b-b15f-377e4e8759d1","podName":"counter-bpbrd-small-group-worker-wffvm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:51:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:52:53.996Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8993a827-5696-462b-b15f-377e4e8759d1","podName":"counter-bpbrd-small-group-worker-wffvm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:51:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:52:53.996Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8993a827-5696-462b-b15f-377e4e8759d1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:52:53.996Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"8993a827-5696-462b-b15f-377e4e8759d1","seconds":300}
{"level":"info","ts":"2026-09-18T09:52:55.267Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d1a49327-77f1-4011-a234-629584cade63","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:55.267Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d1a49327-77f1-4011-a234-629584cade63","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:55.267Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d1a49327-77f1-4011-a234-629584cade63","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:55.267Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d1a49327-77f1-4011-a234-629584cade63"}
{"level":"info","ts":"2026-09-18T09:52:55.267Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d1a49327-77f1-4011-a234-629584cade63","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:55.267Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"d1a49327-77f1-4011-a234-629584cade63","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:52:56.383Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"6189114e-4299-4bcf-af6a-3faee6f70685","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:56.383Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"6189114e-4299-4bcf-af6a-3faee6f70685","pod":"counter-bpbrd-head-vzmvg","ip":"10.132.0.36"}
{"level":"info","ts":"2026-09-18T09:52:56.383Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"6189114e-4299-4bcf-af6a-3faee6f70685","pod":"counter-bpbrd-small-group-worker-wffvm","ip":"10.133.0.36"}
{"level":"info","ts":"2026-09-18T09:52:56.383Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"6189114e-4299-4bcf-af6a-3faee6f70685","rayCluster":"counter-bpbrd","podIPs":["10.132.0.36","10.133.0.36"]}
{"level":"info","ts":"2026-09-18T09:52:58.267Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2e5ec530-038b-4bad-8842-06679b5a0cc9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:52:58.267Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2e5ec530-038b-4bad-8842-06679b5a0cc9","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:52:58.267Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2e5ec530-038b-4bad-8842-06679b5a0cc9","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:52:58.268Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2e5ec530-038b-4bad-8842-06679b5a0cc9"}
{"level":"info","ts":"2026-09-18T09:52:58.268Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2e5ec530-038b-4bad-8842-06679b5a0cc9","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:52:58.268Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2e5ec530-038b-4bad-8842-06679b5a0cc9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:01.268Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"df9df751-b63b-4b22-8f4c-8d1dd2818d32","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:01.268Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"df9df751-b63b-4b22-8f4c-8d1dd2818d32","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:01.268Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"df9df751-b63b-4b22-8f4c-8d1dd2818d32","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:01.268Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"df9df751-b63b-4b22-8f4c-8d1dd2818d32"}
{"level":"info","ts":"2026-09-18T09:53:01.268Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"df9df751-b63b-4b22-8f4c-8d1dd2818d32","RayCluster":"counter-bpbrd","State":""}
{"level":"info","ts":"2026-09-18T09:53:01.268Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"df9df751-b63b-4b22-8f4c-8d1dd2818d32","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:02.994Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"593180b1-4f81-462f-908f-088ea2ec0d15"}
{"level":"info","ts":"2026-09-18T09:53:02.994Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"593180b1-4f81-462f-908f-088ea2ec0d15"}
{"level":"info","ts":"2026-09-18T09:53:02.994Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"593180b1-4f81-462f-908f-088ea2ec0d15"}
{"level":"info","ts":"2026-09-18T09:53:02.994Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"593180b1-4f81-462f-908f-088ea2ec0d15","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:02.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"593180b1-4f81-462f-908f-088ea2ec0d15","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:53:02.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"593180b1-4f81-462f-908f-088ea2ec0d15","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:02.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"593180b1-4f81-462f-908f-088ea2ec0d15","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:53:02.995Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"593180b1-4f81-462f-908f-088ea2ec0d15","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:02.995Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"593180b1-4f81-462f-908f-088ea2ec0d15","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:02.995Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"593180b1-4f81-462f-908f-088ea2ec0d15","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:53:02.995Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"593180b1-4f81-462f-908f-088ea2ec0d15","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:53:02.995Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"593180b1-4f81-462f-908f-088ea2ec0d15","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:02.995Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"593180b1-4f81-462f-908f-088ea2ec0d15","name":"counter-bpbrd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:53:03.009Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"a255cfdc-217a-449e-af39-fe9c4a185e57","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:03.009Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"072962ae-33dc-444e-833b-5a794b4b1dfd","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:03.009Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"072962ae-33dc-444e-833b-5a794b4b1dfd","pod":"counter-bpbrd-head-vzmvg","ip":"10.132.0.36"}
{"level":"info","ts":"2026-09-18T09:53:03.009Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"072962ae-33dc-444e-833b-5a794b4b1dfd","pod":"counter-bpbrd-small-group-worker-wffvm","ip":"10.133.0.36"}
{"level":"info","ts":"2026-09-18T09:53:03.009Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"072962ae-33dc-444e-833b-5a794b4b1dfd","rayCluster":"counter-bpbrd","podIPs":["10.132.0.36","10.133.0.36"]}
{"level":"info","ts":"2026-09-18T09:53:03.009Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a8bdb4b7-e405-48fe-88a5-5e330f5800c6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:03.009Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a8bdb4b7-e405-48fe-88a5-5e330f5800c6","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:03.009Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a8bdb4b7-e405-48fe-88a5-5e330f5800c6","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:03.010Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a8bdb4b7-e405-48fe-88a5-5e330f5800c6"}
{"level":"info","ts":"2026-09-18T09:53:03.029Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a8bdb4b7-e405-48fe-88a5-5e330f5800c6","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-09-18T09:53:03.029Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a8bdb4b7-e405-48fe-88a5-5e330f5800c6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:03.029Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a8bdb4b7-e405-48fe-88a5-5e330f5800c6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:53:03.033Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"a255cfdc-217a-449e-af39-fe9c4a185e57","name":"counter-bpbrd-head"}
{"level":"info","ts":"2026-09-18T09:53:03.046Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6f7afeb6-e1a1-40b3-a3a0-69414f41ea81","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:03.046Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6f7afeb6-e1a1-40b3-a3a0-69414f41ea81","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:03.046Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6f7afeb6-e1a1-40b3-a3a0-69414f41ea81","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:03.046Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6f7afeb6-e1a1-40b3-a3a0-69414f41ea81"}
{"level":"info","ts":"2026-09-18T09:53:03.046Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6f7afeb6-e1a1-40b3-a3a0-69414f41ea81","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:53:03.059Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"a255cfdc-217a-449e-af39-fe9c4a185e57","name":"counter-bpbrd-workers"}
{"level":"info","ts":"2026-09-18T09:53:03.059Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"a255cfdc-217a-449e-af39-fe9c4a185e57","cluster":"counter-bpbrd"}
{"level":"error","ts":"2026-09-18T09:53:03.063Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6f7afeb6-e1a1-40b3-a3a0-69414f41ea81","JobId":"counter-4jgt5","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:03.063Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6f7afeb6-e1a1-40b3-a3a0-69414f41ea81"}
{"level":"error","ts":"2026-09-18T09:53:03.063Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"6f7afeb6-e1a1-40b3-a3a0-69414f41ea81","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:03.063Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a4d1fdaf-83cf-4513-a6fa-248cd31e36b5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:03.063Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a4d1fdaf-83cf-4513-a6fa-248cd31e36b5","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:03.063Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a4d1fdaf-83cf-4513-a6fa-248cd31e36b5","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:03.063Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a4d1fdaf-83cf-4513-a6fa-248cd31e36b5"}
{"level":"info","ts":"2026-09-18T09:53:03.064Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a4d1fdaf-83cf-4513-a6fa-248cd31e36b5","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:53:03.069Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a4d1fdaf-83cf-4513-a6fa-248cd31e36b5","JobId":"counter-4jgt5","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:03.069Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a4d1fdaf-83cf-4513-a6fa-248cd31e36b5"}
{"level":"error","ts":"2026-09-18T09:53:03.069Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a4d1fdaf-83cf-4513-a6fa-248cd31e36b5","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:03.069Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8d2a186d-7159-46b7-aee0-77b79019bff3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:03.069Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8d2a186d-7159-46b7-aee0-77b79019bff3","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:03.069Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8d2a186d-7159-46b7-aee0-77b79019bff3","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:03.069Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8d2a186d-7159-46b7-aee0-77b79019bff3"}
{"level":"info","ts":"2026-09-18T09:53:03.069Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8d2a186d-7159-46b7-aee0-77b79019bff3","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:53:03.074Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8d2a186d-7159-46b7-aee0-77b79019bff3","JobId":"counter-4jgt5","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:03.074Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8d2a186d-7159-46b7-aee0-77b79019bff3"}
{"level":"error","ts":"2026-09-18T09:53:03.074Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8d2a186d-7159-46b7-aee0-77b79019bff3","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:03.079Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e5a22117-5783-4e2b-86a4-e275edfb62cd","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:03.079Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e5a22117-5783-4e2b-86a4-e275edfb62cd","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:03.079Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e5a22117-5783-4e2b-86a4-e275edfb62cd","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:03.079Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e5a22117-5783-4e2b-86a4-e275edfb62cd"}
{"level":"info","ts":"2026-09-18T09:53:03.079Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e5a22117-5783-4e2b-86a4-e275edfb62cd","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:53:03.084Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e5a22117-5783-4e2b-86a4-e275edfb62cd","JobId":"counter-4jgt5","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:03.084Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e5a22117-5783-4e2b-86a4-e275edfb62cd"}
{"level":"error","ts":"2026-09-18T09:53:03.084Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e5a22117-5783-4e2b-86a4-e275edfb62cd","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:03.124Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8304c69c-1d4a-47ba-b1fd-33b4676f4fca","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:03.124Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8304c69c-1d4a-47ba-b1fd-33b4676f4fca","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:03.124Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8304c69c-1d4a-47ba-b1fd-33b4676f4fca","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:03.124Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8304c69c-1d4a-47ba-b1fd-33b4676f4fca"}
{"level":"info","ts":"2026-09-18T09:53:03.124Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8304c69c-1d4a-47ba-b1fd-33b4676f4fca","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:53:03.129Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8304c69c-1d4a-47ba-b1fd-33b4676f4fca","JobId":"counter-4jgt5","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:03.129Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8304c69c-1d4a-47ba-b1fd-33b4676f4fca"}
{"level":"error","ts":"2026-09-18T09:53:03.129Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"8304c69c-1d4a-47ba-b1fd-33b4676f4fca","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:03.210Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"622b2a78-789b-4230-bb6b-8dc3dd72d46f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:03.210Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"622b2a78-789b-4230-bb6b-8dc3dd72d46f","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:03.210Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"622b2a78-789b-4230-bb6b-8dc3dd72d46f","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:03.210Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"622b2a78-789b-4230-bb6b-8dc3dd72d46f"}
{"level":"info","ts":"2026-09-18T09:53:03.210Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"622b2a78-789b-4230-bb6b-8dc3dd72d46f","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:53:03.215Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"622b2a78-789b-4230-bb6b-8dc3dd72d46f","JobId":"counter-4jgt5","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:03.215Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"622b2a78-789b-4230-bb6b-8dc3dd72d46f"}
{"level":"error","ts":"2026-09-18T09:53:03.215Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"622b2a78-789b-4230-bb6b-8dc3dd72d46f","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:03.376Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"12f73022-f342-405b-8d77-459647980e62","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:03.376Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"12f73022-f342-405b-8d77-459647980e62","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:03.376Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"12f73022-f342-405b-8d77-459647980e62","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:03.376Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"12f73022-f342-405b-8d77-459647980e62"}
{"level":"info","ts":"2026-09-18T09:53:03.376Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"12f73022-f342-405b-8d77-459647980e62","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:53:03.382Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"12f73022-f342-405b-8d77-459647980e62","JobId":"counter-4jgt5","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:03.382Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"12f73022-f342-405b-8d77-459647980e62"}
{"level":"error","ts":"2026-09-18T09:53:03.382Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"12f73022-f342-405b-8d77-459647980e62","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:03.703Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"01515d01-e0bd-4e76-b8b3-562d70fae321","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:03.703Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"01515d01-e0bd-4e76-b8b3-562d70fae321","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:03.703Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"01515d01-e0bd-4e76-b8b3-562d70fae321","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:03.703Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"01515d01-e0bd-4e76-b8b3-562d70fae321"}
{"level":"info","ts":"2026-09-18T09:53:03.703Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"01515d01-e0bd-4e76-b8b3-562d70fae321","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:53:03.709Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"01515d01-e0bd-4e76-b8b3-562d70fae321","JobId":"counter-4jgt5","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:03.709Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"01515d01-e0bd-4e76-b8b3-562d70fae321"}
{"level":"error","ts":"2026-09-18T09:53:03.709Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"01515d01-e0bd-4e76-b8b3-562d70fae321","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:04.066Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4ed1bf64-d9a0-4860-9efe-042d7a7cd228","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:04.066Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4ed1bf64-d9a0-4860-9efe-042d7a7cd228","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:04.066Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4ed1bf64-d9a0-4860-9efe-042d7a7cd228","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:04.066Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4ed1bf64-d9a0-4860-9efe-042d7a7cd228"}
{"level":"info","ts":"2026-09-18T09:53:04.066Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4ed1bf64-d9a0-4860-9efe-042d7a7cd228","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:53:04.072Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4ed1bf64-d9a0-4860-9efe-042d7a7cd228","JobId":"counter-4jgt5","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:04.072Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4ed1bf64-d9a0-4860-9efe-042d7a7cd228"}
{"level":"error","ts":"2026-09-18T09:53:04.072Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"4ed1bf64-d9a0-4860-9efe-042d7a7cd228","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:04.350Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"1d15f73d-9b7f-4c61-85c8-9cc960f3ef8e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:04.350Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"1d15f73d-9b7f-4c61-85c8-9cc960f3ef8e","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:04.350Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"1d15f73d-9b7f-4c61-85c8-9cc960f3ef8e","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:04.350Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"1d15f73d-9b7f-4c61-85c8-9cc960f3ef8e"}
{"level":"info","ts":"2026-09-18T09:53:04.350Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"1d15f73d-9b7f-4c61-85c8-9cc960f3ef8e","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:53:04.355Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"1d15f73d-9b7f-4c61-85c8-9cc960f3ef8e","JobId":"counter-4jgt5","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:04.355Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"1d15f73d-9b7f-4c61-85c8-9cc960f3ef8e"}
{"level":"error","ts":"2026-09-18T09:53:04.355Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"1d15f73d-9b7f-4c61-85c8-9cc960f3ef8e","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:05.010Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"1a70b9c6-14e9-4a88-b346-b2565eda6403"}
{"level":"info","ts":"2026-09-18T09:53:05.010Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"1a70b9c6-14e9-4a88-b346-b2565eda6403"}
{"level":"info","ts":"2026-09-18T09:53:05.010Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"1a70b9c6-14e9-4a88-b346-b2565eda6403"}
{"level":"info","ts":"2026-09-18T09:53:05.010Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"1a70b9c6-14e9-4a88-b346-b2565eda6403","1 head service found":"counter-bpbrd-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:05.010Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"1a70b9c6-14e9-4a88-b346-b2565eda6403","Found 1 head Pod":"counter-bpbrd-head-vzmvg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:53:05.010Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"1a70b9c6-14e9-4a88-b346-b2565eda6403","head Pod":"counter-bpbrd-head-vzmvg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-bpbrd-head-vzmvg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:05.010Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"1a70b9c6-14e9-4a88-b346-b2565eda6403","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:53:05.010Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"1a70b9c6-14e9-4a88-b346-b2565eda6403","worker Pod":"counter-bpbrd-small-group-worker-wffvm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-bpbrd-small-group-worker-wffvm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:05.010Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"1a70b9c6-14e9-4a88-b346-b2565eda6403","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:05.010Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"1a70b9c6-14e9-4a88-b346-b2565eda6403","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:53:05.010Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"1a70b9c6-14e9-4a88-b346-b2565eda6403","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:53:05.010Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"1a70b9c6-14e9-4a88-b346-b2565eda6403","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:05.011Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"1a70b9c6-14e9-4a88-b346-b2565eda6403","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:53:05.011Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"1a70b9c6-14e9-4a88-b346-b2565eda6403","seconds":300}
{"level":"info","ts":"2026-09-18T09:53:06.916Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"0fd9c041-6558-410a-94cc-9d4bc6a4686b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:06.916Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"0fd9c041-6558-410a-94cc-9d4bc6a4686b","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:06.916Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"0fd9c041-6558-410a-94cc-9d4bc6a4686b","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:06.916Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"0fd9c041-6558-410a-94cc-9d4bc6a4686b"}
{"level":"info","ts":"2026-09-18T09:53:06.916Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"0fd9c041-6558-410a-94cc-9d4bc6a4686b","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:53:06.921Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"0fd9c041-6558-410a-94cc-9d4bc6a4686b","JobId":"counter-4jgt5","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:06.921Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"0fd9c041-6558-410a-94cc-9d4bc6a4686b"}
{"level":"error","ts":"2026-09-18T09:53:06.921Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"0fd9c041-6558-410a-94cc-9d4bc6a4686b","error":"Job counter-4jgt5 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:12.042Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"09cb9c36-25f0-40e1-8007-b7c970c1db9a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:12.042Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"09cb9c36-25f0-40e1-8007-b7c970c1db9a","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:12.042Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"09cb9c36-25f0-40e1-8007-b7c970c1db9a","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:12.042Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"09cb9c36-25f0-40e1-8007-b7c970c1db9a"}
{"level":"info","ts":"2026-09-18T09:53:12.042Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"09cb9c36-25f0-40e1-8007-b7c970c1db9a","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:53:12.048Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"09cb9c36-25f0-40e1-8007-b7c970c1db9a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:12.048Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"09cb9c36-25f0-40e1-8007-b7c970c1db9a","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:53:12.166Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a8bce1f1-fced-4988-b716-df9df5b14f4d","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:12.166Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a8bce1f1-fced-4988-b716-df9df5b14f4d","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:12.166Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a8bce1f1-fced-4988-b716-df9df5b14f4d","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:12.166Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a8bce1f1-fced-4988-b716-df9df5b14f4d"}
{"level":"info","ts":"2026-09-18T09:53:12.166Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a8bce1f1-fced-4988-b716-df9df5b14f4d","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:53:12.169Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"a8bce1f1-fced-4988-b716-df9df5b14f4d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:15.070Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"55b29ea0-10ab-4b57-a764-af36963f8e1d","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:15.070Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"55b29ea0-10ab-4b57-a764-af36963f8e1d","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:15.071Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"55b29ea0-10ab-4b57-a764-af36963f8e1d","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:15.071Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"55b29ea0-10ab-4b57-a764-af36963f8e1d"}
{"level":"info","ts":"2026-09-18T09:53:15.071Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"55b29ea0-10ab-4b57-a764-af36963f8e1d","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:53:15.075Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"55b29ea0-10ab-4b57-a764-af36963f8e1d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z","endTime":"2026-09-18T09:53:14Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:15.075Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"55b29ea0-10ab-4b57-a764-af36963f8e1d","old JobStatus":"PENDING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:53:15.090Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"27179b7a-fd7d-43bd-a4d8-ff192434edd0","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:15.090Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"27179b7a-fd7d-43bd-a4d8-ff192434edd0","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:15.090Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"27179b7a-fd7d-43bd-a4d8-ff192434edd0","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:15.090Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"27179b7a-fd7d-43bd-a4d8-ff192434edd0"}
{"level":"info","ts":"2026-09-18T09:53:15.090Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"27179b7a-fd7d-43bd-a4d8-ff192434edd0","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:53:15.094Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"27179b7a-fd7d-43bd-a4d8-ff192434edd0","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z","endTime":"2026-09-18T09:53:14Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z","endTime":"2026-09-18T09:53:14Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:18.089Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"58810909-3825-475d-8ae3-4f01a4f96f50","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:18.089Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"58810909-3825-475d-8ae3-4f01a4f96f50","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:18.090Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"58810909-3825-475d-8ae3-4f01a4f96f50","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:18.090Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"58810909-3825-475d-8ae3-4f01a4f96f50"}
{"level":"info","ts":"2026-09-18T09:53:18.090Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"58810909-3825-475d-8ae3-4f01a4f96f50","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:53:18.093Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"58810909-3825-475d-8ae3-4f01a4f96f50","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z","endTime":"2026-09-18T09:53:14Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z","endTime":"2026-09-18T09:53:14Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:21.093Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"aa6f4ee4-f6b9-41d6-ac72-e6a5f2406d49","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:21.094Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"aa6f4ee4-f6b9-41d6-ac72-e6a5f2406d49","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:21.094Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"aa6f4ee4-f6b9-41d6-ac72-e6a5f2406d49","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:21.094Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"aa6f4ee4-f6b9-41d6-ac72-e6a5f2406d49"}
{"level":"info","ts":"2026-09-18T09:53:21.094Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"aa6f4ee4-f6b9-41d6-ac72-e6a5f2406d49","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:53:21.097Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"aa6f4ee4-f6b9-41d6-ac72-e6a5f2406d49","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z","endTime":"2026-09-18T09:53:14Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z","endTime":"2026-09-18T09:53:14Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:22.103Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2c0e35f9-03fa-4a7e-9209-1e8cb8b7b4f7","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:22.103Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2c0e35f9-03fa-4a7e-9209-1e8cb8b7b4f7","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:22.103Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2c0e35f9-03fa-4a7e-9209-1e8cb8b7b4f7","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:22.103Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2c0e35f9-03fa-4a7e-9209-1e8cb8b7b4f7"}
{"level":"info","ts":"2026-09-18T09:53:22.103Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2c0e35f9-03fa-4a7e-9209-1e8cb8b7b4f7","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:53:22.106Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2c0e35f9-03fa-4a7e-9209-1e8cb8b7b4f7","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z","endTime":"2026-09-18T09:53:14Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z","endTime":"2026-09-18T09:53:14Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:23.114Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e04e2eab-8684-4474-82ee-5b06dd131290","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:23.114Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e04e2eab-8684-4474-82ee-5b06dd131290","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:23.114Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e04e2eab-8684-4474-82ee-5b06dd131290","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:23.114Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e04e2eab-8684-4474-82ee-5b06dd131290"}
{"level":"info","ts":"2026-09-18T09:53:23.114Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e04e2eab-8684-4474-82ee-5b06dd131290","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:53:23.118Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"e04e2eab-8684-4474-82ee-5b06dd131290","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z","endTime":"2026-09-18T09:53:14Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z","endTime":"2026-09-18T09:53:14Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:23.133Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7e8802c6-8a64-470c-b5c0-3e6df220ab91","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:23.133Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7e8802c6-8a64-470c-b5c0-3e6df220ab91","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:23.133Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7e8802c6-8a64-470c-b5c0-3e6df220ab91","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:23.133Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7e8802c6-8a64-470c-b5c0-3e6df220ab91"}
{"level":"info","ts":"2026-09-18T09:53:23.133Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7e8802c6-8a64-470c-b5c0-3e6df220ab91","rayCluster":"counter-bpbrd","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:53:23.135Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7e8802c6-8a64-470c-b5c0-3e6df220ab91","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z","endTime":"2026-09-18T09:53:14Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:51:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:53:11Z","endTime":"2026-09-18T09:53:14Z"},"jobId":"counter-4jgt5","rayClusterName":"counter-bpbrd","dashboardURL":"counter-bpbrd-head-svc.test-ns-chvpt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-09-18T09:51:26Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:02Z","stateTransitionTimes":{"ready":"2026-09-18T09:53:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"counter-bpbrd-head-vzmvg","serviceName":"counter-bpbrd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:51:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:53:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:51:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:51:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:23.135Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"7e8802c6-8a64-470c-b5c0-3e6df220ab91","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-09-18T09:53:23.156Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"9d6276f3-faba-4a0f-b4e8-beb8d752c96b","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:23.156Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"9d6276f3-faba-4a0f-b4e8-beb8d752c96b","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:53:23 +0000 UTC","Now":"2026-09-18T09:53:23.156Z","ShutdownTime":"2026-09-18T09:53:23.000Z"}
{"level":"info","ts":"2026-09-18T09:53:23.167Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"9d6276f3-faba-4a0f-b4e8-beb8d752c96b","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:23.167Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"9d6276f3-faba-4a0f-b4e8-beb8d752c96b","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T09:53:23.167Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"9d6276f3-faba-4a0f-b4e8-beb8d752c96b","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:23.167Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"cd912535-d615-4588-bf50-e25b581a7bc4","rayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:23.167Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f2190d6d-4c51-4215-a496-47a70693df03","namespacedName":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:23.167Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f2190d6d-4c51-4215-a496-47a70693df03","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:23.167Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f2190d6d-4c51-4215-a496-47a70693df03","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:23.167Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f2190d6d-4c51-4215-a496-47a70693df03","namespace":"test-ns-chvpt","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T09:53:23.167Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"cd912535-d615-4588-bf50-e25b581a7bc4","pod":"counter-bpbrd-head-vzmvg","ip":"10.132.0.36"}
{"level":"info","ts":"2026-09-18T09:53:23.167Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"cd912535-d615-4588-bf50-e25b581a7bc4","pod":"counter-bpbrd-small-group-worker-wffvm","ip":"10.133.0.36"}
{"level":"info","ts":"2026-09-18T09:53:23.167Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"cd912535-d615-4588-bf50-e25b581a7bc4","rayCluster":"counter-bpbrd","podIPs":["10.132.0.36","10.133.0.36"]}
{"level":"info","ts":"2026-09-18T09:53:23.167Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"reconcileID":"94e07b00-d19c-4d32-9869-b826c091be97"}
{"level":"info","ts":"2026-09-18T09:53:23.167Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"5bc44edf-a03e-4211-b90c-d838466c5daa"}
{"level":"info","ts":"2026-09-18T09:53:23.168Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"683032eb-c66c-41c2-906f-cc088a69aa0f","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:23.168Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"683032eb-c66c-41c2-906f-cc088a69aa0f","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:53:23 +0000 UTC","Now":"2026-09-18T09:53:23.168Z","ShutdownTime":"2026-09-18T09:53:23.000Z"}
{"level":"info","ts":"2026-09-18T09:53:23.168Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"683032eb-c66c-41c2-906f-cc088a69aa0f","RayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:23.168Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"683032eb-c66c-41c2-906f-cc088a69aa0f","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T09:53:23.168Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"683032eb-c66c-41c2-906f-cc088a69aa0f","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:23.173Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f2190d6d-4c51-4215-a496-47a70693df03","name":"kuberay-gateway-access","namespace":"test-ns-chvpt"}
{"level":"info","ts":"2026-09-18T09:53:23.178Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f2190d6d-4c51-4215-a496-47a70693df03","name":"test-ns-chvpt-counter-bpbrd","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T09:53:23.183Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f2190d6d-4c51-4215-a496-47a70693df03","configMap":"kube-rbac-proxy-config-counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:23.188Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f2190d6d-4c51-4215-a496-47a70693df03","serviceAccount":"counter-bpbrd-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:53:23.216Z","logger":"raycluster-resource","msg":"default","name":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:23.216Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-bpbrd","namespace":"test-ns-chvpt"}
{"level":"info","ts":"2026-09-18T09:53:23.227Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"a3a2de04-7f14-4e6a-8c8f-0fc9b6d97e4d"}
{"level":"info","ts":"2026-09-18T09:53:23.227Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"2c763820-4e03-4439-a328-9e1be796c1cc","namespace":"test-ns-chvpt","clusterName":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:23.228Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2aa791df-af7b-4ab0-b5b2-3a182bfa52f3","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:23.228Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2aa791df-af7b-4ab0-b5b2-3a182bfa52f3","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:53:23 +0000 UTC","Now":"2026-09-18T09:53:23.228Z","ShutdownTime":"2026-09-18T09:53:23.000Z"}
{"level":"info","ts":"2026-09-18T09:53:23.228Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2aa791df-af7b-4ab0-b5b2-3a182bfa52f3","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:23.228Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2aa791df-af7b-4ab0-b5b2-3a182bfa52f3","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T09:53:23.228Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2aa791df-af7b-4ab0-b5b2-3a182bfa52f3","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:23.239Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e173f392-1c47-41df-9df8-ae911beed6fa"}
{"level":"info","ts":"2026-09-18T09:53:23.267Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f2190d6d-4c51-4215-a496-47a70693df03","cluster":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:23.267Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f29d6c2a-60d8-4c06-b422-21378b6c439c","namespacedName":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"}}
{"level":"info","ts":"2026-09-18T09:53:23.267Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f29d6c2a-60d8-4c06-b422-21378b6c439c"}
{"level":"info","ts":"2026-09-18T09:53:23.267Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f29d6c2a-60d8-4c06-b422-21378b6c439c","namespace":"test-ns-chvpt","authClusters":0}
{"level":"info","ts":"2026-09-18T09:53:23.267Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"f29d6c2a-60d8-4c06-b422-21378b6c439c","namespace":"test-ns-chvpt"}
{"level":"info","ts":"2026-09-18T09:53:23.268Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"72d168a1-bf7d-42ca-8e7e-bcc6a7fce183"}
{"level":"error","ts":"2026-09-18T09:53:23.268Z","msg":"Failed to delete CA issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"2c763820-4e03-4439-a328-9e1be796c1cc","issuer":"ray-ca-issuer-counter-bpbrd","error":"issuers.cert-manager.io \"ray-ca-issuer-counter-bpbrd\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:247\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:23.296Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"2c763820-4e03-4439-a328-9e1be796c1cc","errorCount":1}
{"level":"info","ts":"2026-09-18T09:53:23.296Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"2c763820-4e03-4439-a328-9e1be796c1cc"}
{"level":"error","ts":"2026-09-18T09:53:23.296Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"2c763820-4e03-4439-a328-9e1be796c1cc","error":"issuers.cert-manager.io \"ray-ca-issuer-counter-bpbrd\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:23.301Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e156aeff-9434-4ecb-8ef4-5039a1612861","namespace":"test-ns-chvpt","clusterName":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:23.301Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-bpbrd","namespace":"test-ns-chvpt"},"namespace":"test-ns-chvpt","name":"counter-bpbrd","reconcileID":"e156aeff-9434-4ecb-8ef4-5039a1612861","namespace":"test-ns-chvpt","clusterName":"counter-bpbrd"}
{"level":"info","ts":"2026-09-18T09:53:24.092Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2e3b2f11-aee3-4b7e-8f38-94d4f94c6b9e","DeletionTimestamp":"2026-09-18 09:53:24 +0000 UTC"}
{"level":"info","ts":"2026-09-18T09:53:24.092Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"2e3b2f11-aee3-4b7e-8f38-94d4f94c6b9e","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:53:24.108Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"bc3918da-c2c3-4a2d-bc85-0d9a9e18f350","DeletionTimestamp":"2026-09-18 09:53:24 +0000 UTC"}
{"level":"info","ts":"2026-09-18T09:53:24.108Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"bc3918da-c2c3-4a2d-bc85-0d9a9e18f350","finalizer":"ray.io/rayjob-finalizer"}
{"level":"error","ts":"2026-09-18T09:53:24.117Z","logger":"controllers.RayJob","msg":"Failed to remove finalizer for RayJob","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"bc3918da-c2c3-4a2d-bc85-0d9a9e18f350","error":"rayjobs.ray.io \"counter\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:143\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:24.117Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"bc3918da-c2c3-4a2d-bc85-0d9a9e18f350"}
{"level":"error","ts":"2026-09-18T09:53:24.117Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"bc3918da-c2c3-4a2d-bc85-0d9a9e18f350","error":"rayjobs.ray.io \"counter\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:24.117Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"b5efa78f-8d90-4319-9f5d-7cad1cfcaafa"}
{"level":"info","ts":"2026-09-18T09:53:24.122Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"53a33d4c-8ee6-4af4-93d1-8dc1fd9aabfc"}
{"level":"info","ts":"2026-09-18T09:53:24.127Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"59d8d128-64d9-46d3-a855-b6bf4993888c"}
{"level":"info","ts":"2026-09-18T09:53:24.127Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter","namespace":"test-ns-chvpt"},"reconcileID":"eb013def-def3-4f81-bd75-1fa3e0dad560"}
{"level":"info","ts":"2026-09-18T09:53:26.177Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"68faf782-44c2-4ba0-8b7d-2c564edfc1f9","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:26.177Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"68faf782-44c2-4ba0-8b7d-2c564edfc1f9","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:53:26.194Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"68faf782-44c2-4ba0-8b7d-2c564edfc1f9"}
{"level":"info","ts":"2026-09-18T09:53:26.194Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"68faf782-44c2-4ba0-8b7d-2c564edfc1f9","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:53:26.194Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"68faf782-44c2-4ba0-8b7d-2c564edfc1f9","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:53:26.194Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"68faf782-44c2-4ba0-8b7d-2c564edfc1f9","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:53:26.209Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ef3824e3-48ca-422b-b704-17d309d96d17","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:26.209Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ef3824e3-48ca-422b-b704-17d309d96d17"}
{"level":"info","ts":"2026-09-18T09:53:26.209Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ef3824e3-48ca-422b-b704-17d309d96d17","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:53:26.209Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ef3824e3-48ca-422b-b704-17d309d96d17","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-vspwj","rayClusterName":"counter-p9h69","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:53:26.209Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ef3824e3-48ca-422b-b704-17d309d96d17","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:53:26.215Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ef3824e3-48ca-422b-b704-17d309d96d17","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T09:53:26.215Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ef3824e3-48ca-422b-b704-17d309d96d17"}
{"level":"error","ts":"2026-09-18T09:53:26.215Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ef3824e3-48ca-422b-b704-17d309d96d17","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.215Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"20126b10-ba89-4b2a-8743-48dcc8fa0f69","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:26.215Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"20126b10-ba89-4b2a-8743-48dcc8fa0f69","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.215Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"20126b10-ba89-4b2a-8743-48dcc8fa0f69","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:26.215Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"20126b10-ba89-4b2a-8743-48dcc8fa0f69","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:26.220Z","logger":"raycluster-resource","msg":"default","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.220Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-fs5bg","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T09:53:26.231Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"ef7f87aa-0fde-4587-ae79-66277f1ec795","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.231Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"982e891b-63f1-4569-be74-b4092a95e57b"}
{"level":"info","ts":"2026-09-18T09:53:26.231Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"982e891b-63f1-4569-be74-b4092a95e57b"}
{"level":"info","ts":"2026-09-18T09:53:26.231Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"982e891b-63f1-4569-be74-b4092a95e57b"}
{"level":"info","ts":"2026-09-18T09:53:26.232Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"47bf4e25-b042-4f4f-9703-ad29b27d855f","namespacedName":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:26.232Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"47bf4e25-b042-4f4f-9703-ad29b27d855f","mode":"IntegratedOAuth","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.232Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"47bf4e25-b042-4f4f-9703-ad29b27d855f","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.232Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"47bf4e25-b042-4f4f-9703-ad29b27d855f","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.232Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"75d190ea-e359-4bed-b5f4-2cd3c77c18ad","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.232Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"75d190ea-e359-4bed-b5f4-2cd3c77c18ad","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.232Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"20126b10-ba89-4b2a-8743-48dcc8fa0f69","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:26.232Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"20126b10-ba89-4b2a-8743-48dcc8fa0f69"}
{"level":"info","ts":"2026-09-18T09:53:26.232Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"20126b10-ba89-4b2a-8743-48dcc8fa0f69","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:26.232Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"20126b10-ba89-4b2a-8743-48dcc8fa0f69","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:53:26.232Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"15385ee2-61ec-478a-bee3-89e8863eae9b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:26.232Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"15385ee2-61ec-478a-bee3-89e8863eae9b","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.233Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"15385ee2-61ec-478a-bee3-89e8863eae9b","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:26.233Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"15385ee2-61ec-478a-bee3-89e8863eae9b"}
{"level":"info","ts":"2026-09-18T09:53:26.233Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"15385ee2-61ec-478a-bee3-89e8863eae9b","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:26.233Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"15385ee2-61ec-478a-bee3-89e8863eae9b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:53:26.240Z","logger":"raycluster-resource","msg":"default","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.240Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-fs5bg","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T09:53:26.266Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"75d190ea-e359-4bed-b5f4-2cd3c77c18ad","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.266Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"75d190ea-e359-4bed-b5f4-2cd3c77c18ad","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.241Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"982e891b-63f1-4569-be74-b4092a95e57b","name":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:26.266Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"982e891b-63f1-4569-be74-b4092a95e57b","error":"head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"info","ts":"2026-09-18T09:53:26.244Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"ef7f87aa-0fde-4587-ae79-66277f1ec795","cluster":"counter-fs5bg"}
{"level":"error","ts":"2026-09-18T09:53:26.266Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"982e891b-63f1-4569-be74-b4092a95e57b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.266Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"982e891b-63f1-4569-be74-b4092a95e57b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"info","ts":"2026-09-18T09:53:26.266Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"bc8649c0-d099-4b92-9958-195878b41b61","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"982e891b-63f1-4569-be74-b4092a95e57b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"982e891b-63f1-4569-be74-b4092a95e57b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"982e891b-63f1-4569-be74-b4092a95e57b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.266Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"982e891b-63f1-4569-be74-b4092a95e57b","name":"counter-fs5bg","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:53:26.290Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"75d190ea-e359-4bed-b5f4-2cd3c77c18ad","rayCluster":"counter-fs5bg","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T09:53:26.290Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"75d190ea-e359-4bed-b5f4-2cd3c77c18ad","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.291Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"9ec14099-5ec8-44d9-a02c-18dff4b86ffd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:26.291Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"9ec14099-5ec8-44d9-a02c-18dff4b86ffd","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.291Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"9ec14099-5ec8-44d9-a02c-18dff4b86ffd","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:26.291Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"9ec14099-5ec8-44d9-a02c-18dff4b86ffd"}
{"level":"info","ts":"2026-09-18T09:53:26.291Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"9ec14099-5ec8-44d9-a02c-18dff4b86ffd","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:26.291Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"9ec14099-5ec8-44d9-a02c-18dff4b86ffd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:53:26.295Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"982e891b-63f1-4569-be74-b4092a95e57b","name":"counter-fs5bg","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-fs5bg\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"counter-fs5bg"}}
{"level":"info","ts":"2026-09-18T09:53:26.295Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"982e891b-63f1-4569-be74-b4092a95e57b"}
{"level":"error","ts":"2026-09-18T09:53:26.295Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"982e891b-63f1-4569-be74-b4092a95e57b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.295Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"ad96a3e2-a13c-4408-9bb9-3e07c41e6314"}
{"level":"info","ts":"2026-09-18T09:53:26.295Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"ad96a3e2-a13c-4408-9bb9-3e07c41e6314"}
{"level":"info","ts":"2026-09-18T09:53:26.295Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"ad96a3e2-a13c-4408-9bb9-3e07c41e6314"}
{"level":"info","ts":"2026-09-18T09:53:26.295Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"ad96a3e2-a13c-4408-9bb9-3e07c41e6314","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:26.295Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"ad96a3e2-a13c-4408-9bb9-3e07c41e6314","error":"head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"error","ts":"2026-09-18T09:53:26.295Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"ad96a3e2-a13c-4408-9bb9-3e07c41e6314","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.295Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"ad96a3e2-a13c-4408-9bb9-3e07c41e6314","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"info","ts":"2026-09-18T09:53:26.296Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"ad96a3e2-a13c-4408-9bb9-3e07c41e6314","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.296Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"ad96a3e2-a13c-4408-9bb9-3e07c41e6314","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.296Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"ad96a3e2-a13c-4408-9bb9-3e07c41e6314","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.296Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"ad96a3e2-a13c-4408-9bb9-3e07c41e6314","name":"counter-fs5bg","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:53:26.298Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"bc8649c0-d099-4b92-9958-195878b41b61","name":"counter-fs5bg-head"}
{"level":"info","ts":"2026-09-18T09:53:26.301Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"47bf4e25-b042-4f4f-9703-ad29b27d855f","name":"counter-fs5bg-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:53:26.311Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"47bf4e25-b042-4f4f-9703-ad29b27d855f","name":"kuberay-gateway-access","namespace":"test-ns-ltx56","operation":"created"}
{"level":"info","ts":"2026-09-18T09:53:26.317Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"ad96a3e2-a13c-4408-9bb9-3e07c41e6314"}
{"level":"error","ts":"2026-09-18T09:53:26.317Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"ad96a3e2-a13c-4408-9bb9-3e07c41e6314","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.317Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"497aa9a8-a5bc-4820-abe2-ae82f1ccb154"}
{"level":"info","ts":"2026-09-18T09:53:26.317Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"497aa9a8-a5bc-4820-abe2-ae82f1ccb154"}
{"level":"info","ts":"2026-09-18T09:53:26.317Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"497aa9a8-a5bc-4820-abe2-ae82f1ccb154"}
{"level":"info","ts":"2026-09-18T09:53:26.317Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"497aa9a8-a5bc-4820-abe2-ae82f1ccb154","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:26.317Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"497aa9a8-a5bc-4820-abe2-ae82f1ccb154","error":"head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"error","ts":"2026-09-18T09:53:26.317Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"497aa9a8-a5bc-4820-abe2-ae82f1ccb154","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.317Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"497aa9a8-a5bc-4820-abe2-ae82f1ccb154","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"info","ts":"2026-09-18T09:53:26.317Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"497aa9a8-a5bc-4820-abe2-ae82f1ccb154","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.317Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"497aa9a8-a5bc-4820-abe2-ae82f1ccb154","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.317Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"497aa9a8-a5bc-4820-abe2-ae82f1ccb154","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.317Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"497aa9a8-a5bc-4820-abe2-ae82f1ccb154","name":"counter-fs5bg","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:53:26.324Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"75d190ea-e359-4bed-b5f4-2cd3c77c18ad","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.331Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"47bf4e25-b042-4f4f-9703-ad29b27d855f","name":"test-ns-ltx56-counter-fs5bg","operation":"created"}
{"level":"info","ts":"2026-09-18T09:53:26.366Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"429e7acb-e232-4e60-8908-3f1745d0bd3a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:26.366Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"429e7acb-e232-4e60-8908-3f1745d0bd3a","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.366Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"429e7acb-e232-4e60-8908-3f1745d0bd3a","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:26.366Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"429e7acb-e232-4e60-8908-3f1745d0bd3a"}
{"level":"info","ts":"2026-09-18T09:53:26.366Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"429e7acb-e232-4e60-8908-3f1745d0bd3a","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:26.366Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"429e7acb-e232-4e60-8908-3f1745d0bd3a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:26.366Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"429e7acb-e232-4e60-8908-3f1745d0bd3a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:53:26.366Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"bc8649c0-d099-4b92-9958-195878b41b61","name":"counter-fs5bg-workers"}
{"level":"info","ts":"2026-09-18T09:53:26.366Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"bc8649c0-d099-4b92-9958-195878b41b61","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.366Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7365ec7f-1e1e-437c-8c7f-be9fcff01fed","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.366Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"497aa9a8-a5bc-4820-abe2-ae82f1ccb154","name":"counter-fs5bg","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-fs5bg\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"counter-fs5bg"}}
{"level":"info","ts":"2026-09-18T09:53:26.366Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"497aa9a8-a5bc-4820-abe2-ae82f1ccb154"}
{"level":"error","ts":"2026-09-18T09:53:26.366Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"497aa9a8-a5bc-4820-abe2-ae82f1ccb154","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.366Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"a3cdde30-5a50-4965-a85a-8534460e6144"}
{"level":"info","ts":"2026-09-18T09:53:26.366Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"a3cdde30-5a50-4965-a85a-8534460e6144"}
{"level":"info","ts":"2026-09-18T09:53:26.366Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"a3cdde30-5a50-4965-a85a-8534460e6144"}
{"level":"info","ts":"2026-09-18T09:53:26.366Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"a3cdde30-5a50-4965-a85a-8534460e6144","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:26.367Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"a3cdde30-5a50-4965-a85a-8534460e6144","error":"head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"error","ts":"2026-09-18T09:53:26.367Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"a3cdde30-5a50-4965-a85a-8534460e6144","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.367Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"a3cdde30-5a50-4965-a85a-8534460e6144","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"info","ts":"2026-09-18T09:53:26.367Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"a3cdde30-5a50-4965-a85a-8534460e6144","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.367Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"a3cdde30-5a50-4965-a85a-8534460e6144","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.367Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"a3cdde30-5a50-4965-a85a-8534460e6144","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.367Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"a3cdde30-5a50-4965-a85a-8534460e6144"}
{"level":"error","ts":"2026-09-18T09:53:26.367Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"a3cdde30-5a50-4965-a85a-8534460e6144","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.372Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"75d190ea-e359-4bed-b5f4-2cd3c77c18ad","rayCluster":"counter-fs5bg","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:53:26.385Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"47bf4e25-b042-4f4f-9703-ad29b27d855f","name":"kube-rbac-proxy-config-counter-fs5bg","operation":"created"}
{"level":"info","ts":"2026-09-18T09:53:26.387Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2ef715f7-43f8-4722-b3e8-81c97fad0102"}
{"level":"info","ts":"2026-09-18T09:53:26.387Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2ef715f7-43f8-4722-b3e8-81c97fad0102"}
{"level":"info","ts":"2026-09-18T09:53:26.387Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2ef715f7-43f8-4722-b3e8-81c97fad0102"}
{"level":"info","ts":"2026-09-18T09:53:26.387Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2ef715f7-43f8-4722-b3e8-81c97fad0102","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:26.387Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2ef715f7-43f8-4722-b3e8-81c97fad0102","error":"head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"error","ts":"2026-09-18T09:53:26.387Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2ef715f7-43f8-4722-b3e8-81c97fad0102","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.387Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2ef715f7-43f8-4722-b3e8-81c97fad0102","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"info","ts":"2026-09-18T09:53:26.387Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2ef715f7-43f8-4722-b3e8-81c97fad0102","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.387Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2ef715f7-43f8-4722-b3e8-81c97fad0102","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.387Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2ef715f7-43f8-4722-b3e8-81c97fad0102","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.387Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2ef715f7-43f8-4722-b3e8-81c97fad0102"}
{"level":"error","ts":"2026-09-18T09:53:26.387Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2ef715f7-43f8-4722-b3e8-81c97fad0102","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.388Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"4a2ea868-95a0-4042-88f4-1eb94542bb0c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:26.388Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"4a2ea868-95a0-4042-88f4-1eb94542bb0c","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.388Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"4a2ea868-95a0-4042-88f4-1eb94542bb0c","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:26.388Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"4a2ea868-95a0-4042-88f4-1eb94542bb0c"}
{"level":"info","ts":"2026-09-18T09:53:26.388Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"4a2ea868-95a0-4042-88f4-1eb94542bb0c","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:26.388Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"4a2ea868-95a0-4042-88f4-1eb94542bb0c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:26.391Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"75d190ea-e359-4bed-b5f4-2cd3c77c18ad","rayCluster":"counter-fs5bg","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:53:26.391Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"75d190ea-e359-4bed-b5f4-2cd3c77c18ad"}
{"level":"info","ts":"2026-09-18T09:53:26.391Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"16a5d6ad-7746-4755-b782-30f33390dcc9","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.391Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"16a5d6ad-7746-4755-b782-30f33390dcc9"}
{"level":"info","ts":"2026-09-18T09:53:26.398Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7365ec7f-1e1e-437c-8c7f-be9fcff01fed","name":"counter-fs5bg-head"}
{"level":"error","ts":"2026-09-18T09:53:26.400Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"47bf4e25-b042-4f4f-9703-ad29b27d855f","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-fs5bg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.401Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"47bf4e25-b042-4f4f-9703-ad29b27d855f"}
{"level":"error","ts":"2026-09-18T09:53:26.401Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"47bf4e25-b042-4f4f-9703-ad29b27d855f","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-fs5bg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.401Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"a394ae39-7de1-49ed-a53d-7981a0c47c3c","namespacedName":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:26.401Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"a394ae39-7de1-49ed-a53d-7981a0c47c3c","mode":"IntegratedOAuth","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.401Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"a394ae39-7de1-49ed-a53d-7981a0c47c3c","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.419Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"a394ae39-7de1-49ed-a53d-7981a0c47c3c","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.419Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"f6eea3ee-e470-4559-a344-c9a4b004ef70","namespacedName":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:26.419Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"f6eea3ee-e470-4559-a344-c9a4b004ef70","mode":"IntegratedOAuth","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.419Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"f6eea3ee-e470-4559-a344-c9a4b004ef70","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.466Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"b6293b5c-eb62-47ea-a16a-12fb76169463","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.466Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"b6293b5c-eb62-47ea-a16a-12fb76169463"}
{"level":"info","ts":"2026-09-18T09:53:26.466Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"0300eb1b-c514-464e-acec-79dd9f676c5e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:26.466Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"0300eb1b-c514-464e-acec-79dd9f676c5e","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.466Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"0300eb1b-c514-464e-acec-79dd9f676c5e","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:26.466Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"0300eb1b-c514-464e-acec-79dd9f676c5e"}
{"level":"info","ts":"2026-09-18T09:53:26.466Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"0300eb1b-c514-464e-acec-79dd9f676c5e","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:26.466Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"0300eb1b-c514-464e-acec-79dd9f676c5e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:26.466Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"0300eb1b-c514-464e-acec-79dd9f676c5e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:53:26.468Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"56eddca9-9d3c-4260-b767-6ceddb9eeeda"}
{"level":"info","ts":"2026-09-18T09:53:26.468Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"56eddca9-9d3c-4260-b767-6ceddb9eeeda"}
{"level":"info","ts":"2026-09-18T09:53:26.468Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"56eddca9-9d3c-4260-b767-6ceddb9eeeda"}
{"level":"info","ts":"2026-09-18T09:53:26.468Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"56eddca9-9d3c-4260-b767-6ceddb9eeeda","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:26.469Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"56eddca9-9d3c-4260-b767-6ceddb9eeeda","error":"head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"error","ts":"2026-09-18T09:53:26.469Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"56eddca9-9d3c-4260-b767-6ceddb9eeeda","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.469Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"56eddca9-9d3c-4260-b767-6ceddb9eeeda","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"info","ts":"2026-09-18T09:53:26.469Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"56eddca9-9d3c-4260-b767-6ceddb9eeeda","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.469Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"56eddca9-9d3c-4260-b767-6ceddb9eeeda","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.469Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"56eddca9-9d3c-4260-b767-6ceddb9eeeda","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.469Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"56eddca9-9d3c-4260-b767-6ceddb9eeeda"}
{"level":"error","ts":"2026-09-18T09:53:26.469Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"56eddca9-9d3c-4260-b767-6ceddb9eeeda","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.487Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7365ec7f-1e1e-437c-8c7f-be9fcff01fed","name":"counter-fs5bg-workers"}
{"level":"info","ts":"2026-09-18T09:53:26.487Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7365ec7f-1e1e-437c-8c7f-be9fcff01fed","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.487Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"6f8c1f0a-daa7-424d-ac36-33e6a926f6c5","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.491Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"f6eea3ee-e470-4559-a344-c9a4b004ef70","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.507Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8212fba8-5fb1-409d-bf13-b3f3aeb19e41","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:26.507Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8212fba8-5fb1-409d-bf13-b3f3aeb19e41","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.507Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8212fba8-5fb1-409d-bf13-b3f3aeb19e41","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:26.508Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8212fba8-5fb1-409d-bf13-b3f3aeb19e41"}
{"level":"info","ts":"2026-09-18T09:53:26.508Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8212fba8-5fb1-409d-bf13-b3f3aeb19e41","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:26.508Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8212fba8-5fb1-409d-bf13-b3f3aeb19e41","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:26.515Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"6f8c1f0a-daa7-424d-ac36-33e6a926f6c5","name":"counter-fs5bg-head"}
{"level":"info","ts":"2026-09-18T09:53:26.527Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"6f8c1f0a-daa7-424d-ac36-33e6a926f6c5","name":"counter-fs5bg-workers"}
{"level":"info","ts":"2026-09-18T09:53:26.527Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"6f8c1f0a-daa7-424d-ac36-33e6a926f6c5","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:26.630Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"47cd2a89-32ea-4724-a491-0e11b3601a7c"}
{"level":"info","ts":"2026-09-18T09:53:26.630Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"47cd2a89-32ea-4724-a491-0e11b3601a7c"}
{"level":"info","ts":"2026-09-18T09:53:26.630Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"47cd2a89-32ea-4724-a491-0e11b3601a7c"}
{"level":"info","ts":"2026-09-18T09:53:26.630Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"47cd2a89-32ea-4724-a491-0e11b3601a7c","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:26.630Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"47cd2a89-32ea-4724-a491-0e11b3601a7c","error":"head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"error","ts":"2026-09-18T09:53:26.630Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"47cd2a89-32ea-4724-a491-0e11b3601a7c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.630Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"47cd2a89-32ea-4724-a491-0e11b3601a7c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"info","ts":"2026-09-18T09:53:26.630Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"47cd2a89-32ea-4724-a491-0e11b3601a7c","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.630Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"47cd2a89-32ea-4724-a491-0e11b3601a7c","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.630Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"47cd2a89-32ea-4724-a491-0e11b3601a7c","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.630Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"47cd2a89-32ea-4724-a491-0e11b3601a7c"}
{"level":"error","ts":"2026-09-18T09:53:26.630Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"47cd2a89-32ea-4724-a491-0e11b3601a7c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.951Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aea75e48-f34d-421e-b6a8-887d2cf97647"}
{"level":"info","ts":"2026-09-18T09:53:26.951Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aea75e48-f34d-421e-b6a8-887d2cf97647"}
{"level":"info","ts":"2026-09-18T09:53:26.951Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aea75e48-f34d-421e-b6a8-887d2cf97647"}
{"level":"info","ts":"2026-09-18T09:53:26.951Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aea75e48-f34d-421e-b6a8-887d2cf97647","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:26.951Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aea75e48-f34d-421e-b6a8-887d2cf97647","error":"head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"error","ts":"2026-09-18T09:53:26.951Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aea75e48-f34d-421e-b6a8-887d2cf97647","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:26.951Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aea75e48-f34d-421e-b6a8-887d2cf97647","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"info","ts":"2026-09-18T09:53:26.951Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aea75e48-f34d-421e-b6a8-887d2cf97647","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.951Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aea75e48-f34d-421e-b6a8-887d2cf97647","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.951Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aea75e48-f34d-421e-b6a8-887d2cf97647","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:26.951Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aea75e48-f34d-421e-b6a8-887d2cf97647"}
{"level":"error","ts":"2026-09-18T09:53:26.951Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aea75e48-f34d-421e-b6a8-887d2cf97647","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:27.592Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c91ee6b8-bff2-473b-bf3f-6e4404537be7"}
{"level":"info","ts":"2026-09-18T09:53:27.592Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c91ee6b8-bff2-473b-bf3f-6e4404537be7"}
{"level":"info","ts":"2026-09-18T09:53:27.592Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c91ee6b8-bff2-473b-bf3f-6e4404537be7"}
{"level":"info","ts":"2026-09-18T09:53:27.592Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c91ee6b8-bff2-473b-bf3f-6e4404537be7","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:27.592Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c91ee6b8-bff2-473b-bf3f-6e4404537be7","error":"head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"error","ts":"2026-09-18T09:53:27.592Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c91ee6b8-bff2-473b-bf3f-6e4404537be7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:27.592Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c91ee6b8-bff2-473b-bf3f-6e4404537be7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"info","ts":"2026-09-18T09:53:27.592Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c91ee6b8-bff2-473b-bf3f-6e4404537be7","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:27.592Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c91ee6b8-bff2-473b-bf3f-6e4404537be7","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:27.592Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c91ee6b8-bff2-473b-bf3f-6e4404537be7","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:27.592Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c91ee6b8-bff2-473b-bf3f-6e4404537be7"}
{"level":"error","ts":"2026-09-18T09:53:27.592Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c91ee6b8-bff2-473b-bf3f-6e4404537be7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:28.873Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6374774c-142e-4d10-8a9c-7ba63b3e8d8b"}
{"level":"info","ts":"2026-09-18T09:53:28.873Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6374774c-142e-4d10-8a9c-7ba63b3e8d8b"}
{"level":"info","ts":"2026-09-18T09:53:28.873Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6374774c-142e-4d10-8a9c-7ba63b3e8d8b"}
{"level":"info","ts":"2026-09-18T09:53:28.873Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6374774c-142e-4d10-8a9c-7ba63b3e8d8b","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:28.873Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6374774c-142e-4d10-8a9c-7ba63b3e8d8b","error":"head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"error","ts":"2026-09-18T09:53:28.873Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6374774c-142e-4d10-8a9c-7ba63b3e8d8b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:28.873Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6374774c-142e-4d10-8a9c-7ba63b3e8d8b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found"}
{"level":"info","ts":"2026-09-18T09:53:28.873Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6374774c-142e-4d10-8a9c-7ba63b3e8d8b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:28.873Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6374774c-142e-4d10-8a9c-7ba63b3e8d8b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:28.873Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6374774c-142e-4d10-8a9c-7ba63b3e8d8b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-fs5bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:53:28.873Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6374774c-142e-4d10-8a9c-7ba63b3e8d8b"}
{"level":"error","ts":"2026-09-18T09:53:28.873Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6374774c-142e-4d10-8a9c-7ba63b3e8d8b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-fs5bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:29.233Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"9285edce-686d-4649-aed7-acb8bcd5232a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:29.233Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"9285edce-686d-4649-aed7-acb8bcd5232a","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:29.233Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"9285edce-686d-4649-aed7-acb8bcd5232a","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:29.233Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"9285edce-686d-4649-aed7-acb8bcd5232a"}
{"level":"info","ts":"2026-09-18T09:53:29.233Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"9285edce-686d-4649-aed7-acb8bcd5232a","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:29.233Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"9285edce-686d-4649-aed7-acb8bcd5232a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:31.434Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc"}
{"level":"info","ts":"2026-09-18T09:53:31.434Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc"}
{"level":"info","ts":"2026-09-18T09:53:31.434Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc"}
{"level":"info","ts":"2026-09-18T09:53:31.434Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:31.434Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc"}
{"level":"info","ts":"2026-09-18T09:53:31.434Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:53:31.434Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc"}
{"level":"info","ts":"2026-09-18T09:53:31.434Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","mode":"IntegratedOAuth","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:31.434Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","cluster":"counter-fs5bg","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:53:31.434Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","cluster":"counter-fs5bg","authType":"IntegratedOAuth","serviceAccount":"counter-fs5bg-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:53:31.434Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-fs5bg","ray.io/group":"headgroup","ray.io/identifier":"counter-fs5bg-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:53:31.434Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:53:31.434Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:53:31.434Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:53:31.434Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","enabled":true}
{"level":"info","ts":"2026-09-18T09:53:31.459Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","name":"counter-fs5bg-head-lbrhk"}
{"level":"info","ts":"2026-09-18T09:53:31.459Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:53:31.459Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:31.459Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:53:31.459Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:53:31.459Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:31.459Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:53:31.459Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc"}
{"level":"info","ts":"2026-09-18T09:53:31.460Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","nodeType":"worker","rayStartParams":{"address":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:53:31.460Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","rayStartCmd":"ray start  --address=counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:53:31.460Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:53:31.460Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","enabled":true}
{"level":"info","ts":"2026-09-18T09:53:31.476Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","name":"counter-fs5bg-small-group-worker-nrzl6"}
{"level":"info","ts":"2026-09-18T09:53:31.476Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:31.476Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:31.476Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"aa5ba43e-6d27-4409-b0aa-8b41552db7fc","name":"counter-fs5bg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"063da88c-b403-4b36-b65f-213024ef0601","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"79f3bb98-cd7e-40ed-8d69-cc0b1f28b5fe"}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"79f3bb98-cd7e-40ed-8d69-cc0b1f28b5fe"}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"79f3bb98-cd7e-40ed-8d69-cc0b1f28b5fe"}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"79f3bb98-cd7e-40ed-8d69-cc0b1f28b5fe","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"063da88c-b403-4b36-b65f-213024ef0601","rayCluster":"counter-fs5bg","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"8485dfdc-897e-4f9e-9f89-e4ad8e78827a","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"894357b4-dd0f-4b84-83e9-b9638e311b0e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"894357b4-dd0f-4b84-83e9-b9638e311b0e","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"79f3bb98-cd7e-40ed-8d69-cc0b1f28b5fe","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"79f3bb98-cd7e-40ed-8d69-cc0b1f28b5fe","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"894357b4-dd0f-4b84-83e9-b9638e311b0e","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"79f3bb98-cd7e-40ed-8d69-cc0b1f28b5fe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"79f3bb98-cd7e-40ed-8d69-cc0b1f28b5fe","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"79f3bb98-cd7e-40ed-8d69-cc0b1f28b5fe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"79f3bb98-cd7e-40ed-8d69-cc0b1f28b5fe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"79f3bb98-cd7e-40ed-8d69-cc0b1f28b5fe","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"79f3bb98-cd7e-40ed-8d69-cc0b1f28b5fe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"894357b4-dd0f-4b84-83e9-b9638e311b0e"}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"894357b4-dd0f-4b84-83e9-b9638e311b0e","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"79f3bb98-cd7e-40ed-8d69-cc0b1f28b5fe","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"894357b4-dd0f-4b84-83e9-b9638e311b0e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"894357b4-dd0f-4b84-83e9-b9638e311b0e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:53:31.490Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"79f3bb98-cd7e-40ed-8d69-cc0b1f28b5fe","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:31.506Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e65a0a44-4363-446e-9e90-cb772f545950","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:31.506Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e65a0a44-4363-446e-9e90-cb772f545950","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:31.506Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e65a0a44-4363-446e-9e90-cb772f545950","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:31.506Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e65a0a44-4363-446e-9e90-cb772f545950"}
{"level":"info","ts":"2026-09-18T09:53:31.506Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e65a0a44-4363-446e-9e90-cb772f545950","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:31.506Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e65a0a44-4363-446e-9e90-cb772f545950","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:31.509Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"8485dfdc-897e-4f9e-9f89-e4ad8e78827a","name":"counter-fs5bg-head"}
{"level":"info","ts":"2026-09-18T09:53:31.521Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"8485dfdc-897e-4f9e-9f89-e4ad8e78827a","name":"counter-fs5bg-workers"}
{"level":"info","ts":"2026-09-18T09:53:31.521Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"8485dfdc-897e-4f9e-9f89-e4ad8e78827a","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"79f3bb98-cd7e-40ed-8d69-cc0b1f28b5fe","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"79f3bb98-cd7e-40ed-8d69-cc0b1f28b5fe","seconds":300}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"98ad07a5-66a8-42cb-b49e-b4d819bd3c9d"}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"98ad07a5-66a8-42cb-b49e-b4d819bd3c9d"}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"98ad07a5-66a8-42cb-b49e-b4d819bd3c9d"}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"98ad07a5-66a8-42cb-b49e-b4d819bd3c9d","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"98ad07a5-66a8-42cb-b49e-b4d819bd3c9d","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"98ad07a5-66a8-42cb-b49e-b4d819bd3c9d","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"98ad07a5-66a8-42cb-b49e-b4d819bd3c9d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"98ad07a5-66a8-42cb-b49e-b4d819bd3c9d","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"98ad07a5-66a8-42cb-b49e-b4d819bd3c9d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"98ad07a5-66a8-42cb-b49e-b4d819bd3c9d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"98ad07a5-66a8-42cb-b49e-b4d819bd3c9d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"98ad07a5-66a8-42cb-b49e-b4d819bd3c9d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"98ad07a5-66a8-42cb-b49e-b4d819bd3c9d","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"98ad07a5-66a8-42cb-b49e-b4d819bd3c9d","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:31.566Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"98ad07a5-66a8-42cb-b49e-b4d819bd3c9d","name":"counter-fs5bg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:53:31.581Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"c188b9a8-7e5c-4de0-8f26-dafcf4ad63cf","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:31.581Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"bfb23341-5d9e-475e-803e-1e8c59bb9687","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:31.581Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"bfb23341-5d9e-475e-803e-1e8c59bb9687","rayCluster":"counter-fs5bg","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:53:31.581Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"cd64ae9e-e9df-40c2-8cfd-1b87eff73293","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:31.581Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"cd64ae9e-e9df-40c2-8cfd-1b87eff73293","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:31.581Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"cd64ae9e-e9df-40c2-8cfd-1b87eff73293","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:31.581Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"cd64ae9e-e9df-40c2-8cfd-1b87eff73293"}
{"level":"info","ts":"2026-09-18T09:53:31.581Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"cd64ae9e-e9df-40c2-8cfd-1b87eff73293","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:31.581Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"cd64ae9e-e9df-40c2-8cfd-1b87eff73293","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:31.581Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"cd64ae9e-e9df-40c2-8cfd-1b87eff73293","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:53:31.591Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"c188b9a8-7e5c-4de0-8f26-dafcf4ad63cf","name":"counter-fs5bg-head"}
{"level":"info","ts":"2026-09-18T09:53:31.596Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fbd4606e-e292-4e7f-a41c-5c6374b38ae5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:31.596Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fbd4606e-e292-4e7f-a41c-5c6374b38ae5","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:31.596Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fbd4606e-e292-4e7f-a41c-5c6374b38ae5","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:31.596Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fbd4606e-e292-4e7f-a41c-5c6374b38ae5"}
{"level":"info","ts":"2026-09-18T09:53:31.596Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fbd4606e-e292-4e7f-a41c-5c6374b38ae5","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:31.596Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fbd4606e-e292-4e7f-a41c-5c6374b38ae5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:31.602Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"c188b9a8-7e5c-4de0-8f26-dafcf4ad63cf","name":"counter-fs5bg-workers"}
{"level":"info","ts":"2026-09-18T09:53:31.602Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"c188b9a8-7e5c-4de0-8f26-dafcf4ad63cf","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:32.055Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e86a1509-1998-45a9-90a8-24f98b85ca7d"}
{"level":"info","ts":"2026-09-18T09:53:32.055Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e86a1509-1998-45a9-90a8-24f98b85ca7d"}
{"level":"info","ts":"2026-09-18T09:53:32.055Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e86a1509-1998-45a9-90a8-24f98b85ca7d"}
{"level":"info","ts":"2026-09-18T09:53:32.055Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e86a1509-1998-45a9-90a8-24f98b85ca7d","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:32.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e86a1509-1998-45a9-90a8-24f98b85ca7d","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:53:32.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e86a1509-1998-45a9-90a8-24f98b85ca7d","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:32.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e86a1509-1998-45a9-90a8-24f98b85ca7d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:53:32.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e86a1509-1998-45a9-90a8-24f98b85ca7d","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:32.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e86a1509-1998-45a9-90a8-24f98b85ca7d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:32.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e86a1509-1998-45a9-90a8-24f98b85ca7d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:53:32.055Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e86a1509-1998-45a9-90a8-24f98b85ca7d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:53:32.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e86a1509-1998-45a9-90a8-24f98b85ca7d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:32.055Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e86a1509-1998-45a9-90a8-24f98b85ca7d","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:32.055Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e86a1509-1998-45a9-90a8-24f98b85ca7d","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:32.055Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e86a1509-1998-45a9-90a8-24f98b85ca7d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:53:32.055Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e86a1509-1998-45a9-90a8-24f98b85ca7d","seconds":300}
{"level":"info","ts":"2026-09-18T09:53:32.077Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06fa3212-2cf7-452e-918e-895a4255144a"}
{"level":"info","ts":"2026-09-18T09:53:32.077Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06fa3212-2cf7-452e-918e-895a4255144a"}
{"level":"info","ts":"2026-09-18T09:53:32.077Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06fa3212-2cf7-452e-918e-895a4255144a"}
{"level":"info","ts":"2026-09-18T09:53:32.077Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06fa3212-2cf7-452e-918e-895a4255144a","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:32.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06fa3212-2cf7-452e-918e-895a4255144a","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:53:32.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06fa3212-2cf7-452e-918e-895a4255144a","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:32.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06fa3212-2cf7-452e-918e-895a4255144a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:53:32.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06fa3212-2cf7-452e-918e-895a4255144a","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:32.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06fa3212-2cf7-452e-918e-895a4255144a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:32.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06fa3212-2cf7-452e-918e-895a4255144a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:53:32.077Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06fa3212-2cf7-452e-918e-895a4255144a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:53:32.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06fa3212-2cf7-452e-918e-895a4255144a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:32.077Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06fa3212-2cf7-452e-918e-895a4255144a","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:32.077Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06fa3212-2cf7-452e-918e-895a4255144a","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:32.077Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06fa3212-2cf7-452e-918e-895a4255144a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:53:32.077Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06fa3212-2cf7-452e-918e-895a4255144a","seconds":300}
{"level":"info","ts":"2026-09-18T09:53:32.147Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e4e8ccd1-e266-4011-a83f-b7a507314456"}
{"level":"info","ts":"2026-09-18T09:53:32.147Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e4e8ccd1-e266-4011-a83f-b7a507314456"}
{"level":"info","ts":"2026-09-18T09:53:32.147Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e4e8ccd1-e266-4011-a83f-b7a507314456"}
{"level":"info","ts":"2026-09-18T09:53:32.147Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e4e8ccd1-e266-4011-a83f-b7a507314456","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:32.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e4e8ccd1-e266-4011-a83f-b7a507314456","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:53:32.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e4e8ccd1-e266-4011-a83f-b7a507314456","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:32.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e4e8ccd1-e266-4011-a83f-b7a507314456","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:53:32.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e4e8ccd1-e266-4011-a83f-b7a507314456","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:32.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e4e8ccd1-e266-4011-a83f-b7a507314456","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:32.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e4e8ccd1-e266-4011-a83f-b7a507314456","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:53:32.147Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e4e8ccd1-e266-4011-a83f-b7a507314456","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:53:32.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e4e8ccd1-e266-4011-a83f-b7a507314456","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:32.147Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e4e8ccd1-e266-4011-a83f-b7a507314456","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:32.147Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e4e8ccd1-e266-4011-a83f-b7a507314456","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:32.148Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e4e8ccd1-e266-4011-a83f-b7a507314456","name":"counter-fs5bg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:53:32.162Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"874c2f56-80e7-4b35-a491-5f032e38d9d8","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:32.162Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"874c2f56-80e7-4b35-a491-5f032e38d9d8","pod":"counter-fs5bg-head-lbrhk","ip":"10.133.0.38"}
{"level":"info","ts":"2026-09-18T09:53:32.162Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"07ed736f-f558-4c20-b9c4-b522896748d9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:32.162Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"07ed736f-f558-4c20-b9c4-b522896748d9","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:32.162Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"07ed736f-f558-4c20-b9c4-b522896748d9","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:32.162Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"07ed736f-f558-4c20-b9c4-b522896748d9"}
{"level":"info","ts":"2026-09-18T09:53:32.162Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"07ed736f-f558-4c20-b9c4-b522896748d9","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:32.162Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"07ed736f-f558-4c20-b9c4-b522896748d9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:32.162Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"07ed736f-f558-4c20-b9c4-b522896748d9","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:53:32.162Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"db378b84-e792-4503-ba50-d88ef03821c2","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:32.175Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"874c2f56-80e7-4b35-a491-5f032e38d9d8","certificate":"ray-head-cert-counter-fs5bg","dnsNames":["counter-fs5bg-head-svc","localhost","counter-fs5bg-head-svc.test-ns-ltx56.svc","counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.38","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:53:32.180Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3ecfb42e-98a3-4781-b13c-bf96b579c125","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:32.180Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3ecfb42e-98a3-4781-b13c-bf96b579c125","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:32.180Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3ecfb42e-98a3-4781-b13c-bf96b579c125","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:32.180Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3ecfb42e-98a3-4781-b13c-bf96b579c125"}
{"level":"info","ts":"2026-09-18T09:53:32.180Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3ecfb42e-98a3-4781-b13c-bf96b579c125","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:32.180Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3ecfb42e-98a3-4781-b13c-bf96b579c125","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:32.182Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"db378b84-e792-4503-ba50-d88ef03821c2","name":"counter-fs5bg-head"}
{"level":"info","ts":"2026-09-18T09:53:32.195Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"874c2f56-80e7-4b35-a491-5f032e38d9d8","certificate":"ray-worker-cert-counter-fs5bg","dnsNames":["counter-fs5bg-worker-svc","localhost","counter-fs5bg-small-group","counter-fs5bg-worker-svc.test-ns-ltx56.svc","counter-fs5bg-worker-svc.test-ns-ltx56.svc.cluster.local","*.counter-fs5bg-worker-svc.test-ns-ltx56.svc","*.counter-fs5bg-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.38","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:53:32.195Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"874c2f56-80e7-4b35-a491-5f032e38d9d8","rayCluster":"counter-fs5bg","podIPs":["10.133.0.38"]}
{"level":"info","ts":"2026-09-18T09:53:32.202Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"db378b84-e792-4503-ba50-d88ef03821c2","name":"counter-fs5bg-workers"}
{"level":"info","ts":"2026-09-18T09:53:32.202Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"db378b84-e792-4503-ba50-d88ef03821c2","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:32.234Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e4b44df1-3f68-46b6-90ae-fbf5bf50b29e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:32.234Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e4b44df1-3f68-46b6-90ae-fbf5bf50b29e","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:32.234Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e4b44df1-3f68-46b6-90ae-fbf5bf50b29e","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:32.234Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e4b44df1-3f68-46b6-90ae-fbf5bf50b29e"}
{"level":"info","ts":"2026-09-18T09:53:32.234Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e4b44df1-3f68-46b6-90ae-fbf5bf50b29e","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:32.234Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e4b44df1-3f68-46b6-90ae-fbf5bf50b29e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:32.461Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"82865bc3-babc-4094-add5-44f215b42485"}
{"level":"info","ts":"2026-09-18T09:53:32.461Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"82865bc3-babc-4094-add5-44f215b42485"}
{"level":"info","ts":"2026-09-18T09:53:32.461Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"82865bc3-babc-4094-add5-44f215b42485"}
{"level":"info","ts":"2026-09-18T09:53:32.461Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"82865bc3-babc-4094-add5-44f215b42485","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:32.461Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"82865bc3-babc-4094-add5-44f215b42485","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:53:32.461Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"82865bc3-babc-4094-add5-44f215b42485","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:32.461Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"82865bc3-babc-4094-add5-44f215b42485","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:53:32.461Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"82865bc3-babc-4094-add5-44f215b42485","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:32.461Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"82865bc3-babc-4094-add5-44f215b42485","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:32.461Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"82865bc3-babc-4094-add5-44f215b42485","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:53:32.461Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"82865bc3-babc-4094-add5-44f215b42485","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:53:32.461Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"82865bc3-babc-4094-add5-44f215b42485","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:32.461Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"82865bc3-babc-4094-add5-44f215b42485","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:32.461Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"82865bc3-babc-4094-add5-44f215b42485","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:32.461Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"82865bc3-babc-4094-add5-44f215b42485","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:53:32.461Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"82865bc3-babc-4094-add5-44f215b42485","seconds":300}
{"level":"info","ts":"2026-09-18T09:53:33.490Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"8d25b641-9414-49df-8e23-28dbe025a2c6"}
{"level":"info","ts":"2026-09-18T09:53:33.490Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"8d25b641-9414-49df-8e23-28dbe025a2c6"}
{"level":"info","ts":"2026-09-18T09:53:33.490Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"8d25b641-9414-49df-8e23-28dbe025a2c6"}
{"level":"info","ts":"2026-09-18T09:53:33.490Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"8d25b641-9414-49df-8e23-28dbe025a2c6","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:33.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"8d25b641-9414-49df-8e23-28dbe025a2c6","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:53:33.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"8d25b641-9414-49df-8e23-28dbe025a2c6","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:33.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"8d25b641-9414-49df-8e23-28dbe025a2c6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:53:33.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"8d25b641-9414-49df-8e23-28dbe025a2c6","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:33.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"8d25b641-9414-49df-8e23-28dbe025a2c6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:33.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"8d25b641-9414-49df-8e23-28dbe025a2c6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:53:33.491Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"8d25b641-9414-49df-8e23-28dbe025a2c6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:53:33.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"8d25b641-9414-49df-8e23-28dbe025a2c6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:33.491Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"8d25b641-9414-49df-8e23-28dbe025a2c6","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:33.491Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"8d25b641-9414-49df-8e23-28dbe025a2c6","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:33.491Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"8d25b641-9414-49df-8e23-28dbe025a2c6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:53:33.491Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"8d25b641-9414-49df-8e23-28dbe025a2c6","seconds":300}
{"level":"info","ts":"2026-09-18T09:53:35.235Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a60ea131-ee23-4f14-bc73-9028dc04123f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:35.235Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a60ea131-ee23-4f14-bc73-9028dc04123f","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:35.235Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a60ea131-ee23-4f14-bc73-9028dc04123f","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:35.235Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a60ea131-ee23-4f14-bc73-9028dc04123f"}
{"level":"info","ts":"2026-09-18T09:53:35.235Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a60ea131-ee23-4f14-bc73-9028dc04123f","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:35.235Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a60ea131-ee23-4f14-bc73-9028dc04123f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:38.171Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"b2fc3ee2-727c-4421-9325-ff43fbe355e6"}
{"level":"info","ts":"2026-09-18T09:53:38.171Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"b2fc3ee2-727c-4421-9325-ff43fbe355e6"}
{"level":"info","ts":"2026-09-18T09:53:38.171Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"b2fc3ee2-727c-4421-9325-ff43fbe355e6"}
{"level":"info","ts":"2026-09-18T09:53:38.171Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"b2fc3ee2-727c-4421-9325-ff43fbe355e6","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:38.171Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"b2fc3ee2-727c-4421-9325-ff43fbe355e6","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:53:38.171Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"b2fc3ee2-727c-4421-9325-ff43fbe355e6","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:38.171Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"b2fc3ee2-727c-4421-9325-ff43fbe355e6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:53:38.171Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"b2fc3ee2-727c-4421-9325-ff43fbe355e6","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:38.171Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"b2fc3ee2-727c-4421-9325-ff43fbe355e6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:38.171Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"b2fc3ee2-727c-4421-9325-ff43fbe355e6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:53:38.171Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"b2fc3ee2-727c-4421-9325-ff43fbe355e6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:53:38.171Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"b2fc3ee2-727c-4421-9325-ff43fbe355e6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:38.171Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"b2fc3ee2-727c-4421-9325-ff43fbe355e6","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:38.171Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"b2fc3ee2-727c-4421-9325-ff43fbe355e6","podName":"counter-fs5bg-head-lbrhk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:38.171Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"b2fc3ee2-727c-4421-9325-ff43fbe355e6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:53:38.171Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"b2fc3ee2-727c-4421-9325-ff43fbe355e6","seconds":300}
{"level":"info","ts":"2026-09-18T09:53:38.236Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c97b83e6-7bb3-4286-abb0-2f758bb592c9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:38.236Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c97b83e6-7bb3-4286-abb0-2f758bb592c9","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:38.236Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c97b83e6-7bb3-4286-abb0-2f758bb592c9","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:38.236Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c97b83e6-7bb3-4286-abb0-2f758bb592c9"}
{"level":"info","ts":"2026-09-18T09:53:38.236Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c97b83e6-7bb3-4286-abb0-2f758bb592c9","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:38.236Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c97b83e6-7bb3-4286-abb0-2f758bb592c9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:39.180Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"4475ed81-dd12-4a73-ac6b-192a6ad34970"}
{"level":"info","ts":"2026-09-18T09:53:39.181Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"4475ed81-dd12-4a73-ac6b-192a6ad34970"}
{"level":"info","ts":"2026-09-18T09:53:39.181Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"4475ed81-dd12-4a73-ac6b-192a6ad34970"}
{"level":"info","ts":"2026-09-18T09:53:39.181Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"4475ed81-dd12-4a73-ac6b-192a6ad34970","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:39.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"4475ed81-dd12-4a73-ac6b-192a6ad34970","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:53:39.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"4475ed81-dd12-4a73-ac6b-192a6ad34970","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:39.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"4475ed81-dd12-4a73-ac6b-192a6ad34970","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:53:39.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"4475ed81-dd12-4a73-ac6b-192a6ad34970","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:39.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"4475ed81-dd12-4a73-ac6b-192a6ad34970","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:39.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"4475ed81-dd12-4a73-ac6b-192a6ad34970","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:53:39.181Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"4475ed81-dd12-4a73-ac6b-192a6ad34970","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:53:39.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"4475ed81-dd12-4a73-ac6b-192a6ad34970","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:39.181Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"4475ed81-dd12-4a73-ac6b-192a6ad34970","podName":"counter-fs5bg-head-lbrhk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:53:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:53:39.181Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"4475ed81-dd12-4a73-ac6b-192a6ad34970","podName":"counter-fs5bg-head-lbrhk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:53:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:53:39.181Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"4475ed81-dd12-4a73-ac6b-192a6ad34970","name":"counter-fs5bg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:53:39.195Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"8123500c-56d9-4fa9-8761-14eac4713d77","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:39.195Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"796176ff-77f1-41db-9476-42b7005a9005","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:39.195Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"796176ff-77f1-41db-9476-42b7005a9005","pod":"counter-fs5bg-head-lbrhk","ip":"10.133.0.38"}
{"level":"info","ts":"2026-09-18T09:53:39.195Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"796176ff-77f1-41db-9476-42b7005a9005","pod":"counter-fs5bg-small-group-worker-nrzl6","ip":"10.132.0.37"}
{"level":"info","ts":"2026-09-18T09:53:39.196Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ba7c50dc-6bac-4656-b29a-7c57821db91c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:39.196Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ba7c50dc-6bac-4656-b29a-7c57821db91c","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:39.196Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ba7c50dc-6bac-4656-b29a-7c57821db91c","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:39.196Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ba7c50dc-6bac-4656-b29a-7c57821db91c"}
{"level":"info","ts":"2026-09-18T09:53:39.196Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ba7c50dc-6bac-4656-b29a-7c57821db91c","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:39.196Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ba7c50dc-6bac-4656-b29a-7c57821db91c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:39.196Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ba7c50dc-6bac-4656-b29a-7c57821db91c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:53:39.269Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"796176ff-77f1-41db-9476-42b7005a9005","certificate":"ray-head-cert-counter-fs5bg","dnsNames":["counter-fs5bg-head-svc","localhost","counter-fs5bg-head-svc.test-ns-ltx56.svc","counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.38","10.132.0.37","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:53:39.270Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"73f9f283-a353-459a-8280-7d04db979a3f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:39.270Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"73f9f283-a353-459a-8280-7d04db979a3f","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:39.270Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"73f9f283-a353-459a-8280-7d04db979a3f","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:39.366Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"8123500c-56d9-4fa9-8761-14eac4713d77","name":"counter-fs5bg-head"}
{"level":"info","ts":"2026-09-18T09:53:39.366Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"796176ff-77f1-41db-9476-42b7005a9005","certificate":"ray-worker-cert-counter-fs5bg","dnsNames":["counter-fs5bg-worker-svc","localhost","counter-fs5bg-small-group","counter-fs5bg-worker-svc.test-ns-ltx56.svc","counter-fs5bg-worker-svc.test-ns-ltx56.svc.cluster.local","*.counter-fs5bg-worker-svc.test-ns-ltx56.svc","*.counter-fs5bg-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.38","10.132.0.37","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:53:39.367Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"796176ff-77f1-41db-9476-42b7005a9005","rayCluster":"counter-fs5bg","podIPs":["10.133.0.38","10.132.0.37"]}
{"level":"info","ts":"2026-09-18T09:53:39.368Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"73f9f283-a353-459a-8280-7d04db979a3f"}
{"level":"info","ts":"2026-09-18T09:53:39.368Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"73f9f283-a353-459a-8280-7d04db979a3f","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:39.368Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"73f9f283-a353-459a-8280-7d04db979a3f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:39.381Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"8123500c-56d9-4fa9-8761-14eac4713d77","name":"counter-fs5bg-workers"}
{"level":"info","ts":"2026-09-18T09:53:39.381Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"8123500c-56d9-4fa9-8761-14eac4713d77","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:41.195Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"434ac9bc-59f2-41e1-84d9-b25c4617a870"}
{"level":"info","ts":"2026-09-18T09:53:41.195Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"434ac9bc-59f2-41e1-84d9-b25c4617a870"}
{"level":"info","ts":"2026-09-18T09:53:41.195Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"434ac9bc-59f2-41e1-84d9-b25c4617a870"}
{"level":"info","ts":"2026-09-18T09:53:41.195Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"434ac9bc-59f2-41e1-84d9-b25c4617a870","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:41.195Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"434ac9bc-59f2-41e1-84d9-b25c4617a870","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:53:41.195Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"434ac9bc-59f2-41e1-84d9-b25c4617a870","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:41.195Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"434ac9bc-59f2-41e1-84d9-b25c4617a870","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:53:41.195Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"434ac9bc-59f2-41e1-84d9-b25c4617a870","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:41.195Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"434ac9bc-59f2-41e1-84d9-b25c4617a870","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:41.195Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"434ac9bc-59f2-41e1-84d9-b25c4617a870","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:53:41.195Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"434ac9bc-59f2-41e1-84d9-b25c4617a870","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:53:41.195Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"434ac9bc-59f2-41e1-84d9-b25c4617a870","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:41.195Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"434ac9bc-59f2-41e1-84d9-b25c4617a870","podName":"counter-fs5bg-head-lbrhk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:53:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:53:41.195Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"434ac9bc-59f2-41e1-84d9-b25c4617a870","podName":"counter-fs5bg-head-lbrhk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:53:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:53:41.196Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"434ac9bc-59f2-41e1-84d9-b25c4617a870","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:53:41.196Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"434ac9bc-59f2-41e1-84d9-b25c4617a870","seconds":300}
{"level":"info","ts":"2026-09-18T09:53:41.237Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ea9b384c-deeb-4e3a-92c6-5bf44efaa1f5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:41.237Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ea9b384c-deeb-4e3a-92c6-5bf44efaa1f5","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:41.237Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ea9b384c-deeb-4e3a-92c6-5bf44efaa1f5","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:41.237Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ea9b384c-deeb-4e3a-92c6-5bf44efaa1f5"}
{"level":"info","ts":"2026-09-18T09:53:41.237Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ea9b384c-deeb-4e3a-92c6-5bf44efaa1f5","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:41.237Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ea9b384c-deeb-4e3a-92c6-5bf44efaa1f5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:44.238Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f6edad81-6536-4602-8cc0-0f7ea542f5f0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:44.238Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f6edad81-6536-4602-8cc0-0f7ea542f5f0","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:44.238Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f6edad81-6536-4602-8cc0-0f7ea542f5f0","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:44.238Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f6edad81-6536-4602-8cc0-0f7ea542f5f0"}
{"level":"info","ts":"2026-09-18T09:53:44.238Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f6edad81-6536-4602-8cc0-0f7ea542f5f0","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:44.238Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f6edad81-6536-4602-8cc0-0f7ea542f5f0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:47.240Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c3bd54c9-a24c-484d-8ad1-fc7d841129ac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:47.240Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c3bd54c9-a24c-484d-8ad1-fc7d841129ac","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:47.240Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c3bd54c9-a24c-484d-8ad1-fc7d841129ac","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:47.240Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c3bd54c9-a24c-484d-8ad1-fc7d841129ac"}
{"level":"info","ts":"2026-09-18T09:53:47.240Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c3bd54c9-a24c-484d-8ad1-fc7d841129ac","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:47.240Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c3bd54c9-a24c-484d-8ad1-fc7d841129ac","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:50.213Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"3f310c77-a555-4d8c-977c-02185a76fd9a"}
{"level":"info","ts":"2026-09-18T09:53:50.213Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"3f310c77-a555-4d8c-977c-02185a76fd9a"}
{"level":"info","ts":"2026-09-18T09:53:50.213Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"3f310c77-a555-4d8c-977c-02185a76fd9a"}
{"level":"info","ts":"2026-09-18T09:53:50.213Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"3f310c77-a555-4d8c-977c-02185a76fd9a","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:50.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"3f310c77-a555-4d8c-977c-02185a76fd9a","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:53:50.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"3f310c77-a555-4d8c-977c-02185a76fd9a","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:50.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"3f310c77-a555-4d8c-977c-02185a76fd9a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:53:50.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"3f310c77-a555-4d8c-977c-02185a76fd9a","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:50.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"3f310c77-a555-4d8c-977c-02185a76fd9a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:50.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"3f310c77-a555-4d8c-977c-02185a76fd9a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:53:50.213Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"3f310c77-a555-4d8c-977c-02185a76fd9a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:53:50.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"3f310c77-a555-4d8c-977c-02185a76fd9a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:50.213Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"3f310c77-a555-4d8c-977c-02185a76fd9a","podName":"counter-fs5bg-small-group-worker-nrzl6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:50.213Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"3f310c77-a555-4d8c-977c-02185a76fd9a","podName":"counter-fs5bg-small-group-worker-nrzl6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:50.214Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"3f310c77-a555-4d8c-977c-02185a76fd9a","name":"counter-fs5bg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:53:50.234Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"b7c2b18c-661b-4d23-86e9-fe11d6c66e35","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:50.234Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7b5864b9-c6a5-423b-9145-597932836c79","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:50.234Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"b7c2b18c-661b-4d23-86e9-fe11d6c66e35","pod":"counter-fs5bg-head-lbrhk","ip":"10.133.0.38"}
{"level":"info","ts":"2026-09-18T09:53:50.234Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"b7c2b18c-661b-4d23-86e9-fe11d6c66e35","pod":"counter-fs5bg-small-group-worker-nrzl6","ip":"10.132.0.37"}
{"level":"info","ts":"2026-09-18T09:53:50.234Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"b7c2b18c-661b-4d23-86e9-fe11d6c66e35","rayCluster":"counter-fs5bg","podIPs":["10.133.0.38","10.132.0.37"]}
{"level":"info","ts":"2026-09-18T09:53:50.234Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f51e200a-103a-4da6-9c4c-cf6795c5810b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:50.234Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f51e200a-103a-4da6-9c4c-cf6795c5810b","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:50.234Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f51e200a-103a-4da6-9c4c-cf6795c5810b","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:50.234Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f51e200a-103a-4da6-9c4c-cf6795c5810b"}
{"level":"info","ts":"2026-09-18T09:53:50.234Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f51e200a-103a-4da6-9c4c-cf6795c5810b","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:50.234Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f51e200a-103a-4da6-9c4c-cf6795c5810b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:50.234Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f51e200a-103a-4da6-9c4c-cf6795c5810b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:53:50.250Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"371d18a9-4b74-4663-9202-ef54ee0014b6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:50.250Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"371d18a9-4b74-4663-9202-ef54ee0014b6","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:50.250Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"371d18a9-4b74-4663-9202-ef54ee0014b6","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:50.250Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"371d18a9-4b74-4663-9202-ef54ee0014b6"}
{"level":"info","ts":"2026-09-18T09:53:50.250Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"371d18a9-4b74-4663-9202-ef54ee0014b6","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:50.250Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"371d18a9-4b74-4663-9202-ef54ee0014b6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:50.250Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"371d18a9-4b74-4663-9202-ef54ee0014b6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:53:50.252Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7b5864b9-c6a5-423b-9145-597932836c79","name":"counter-fs5bg-head"}
{"level":"info","ts":"2026-09-18T09:53:50.257Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"371d18a9-4b74-4663-9202-ef54ee0014b6","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T09:53:50.257Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"371d18a9-4b74-4663-9202-ef54ee0014b6"}
{"level":"error","ts":"2026-09-18T09:53:50.257Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"371d18a9-4b74-4663-9202-ef54ee0014b6","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:53:50.257Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7ff9a888-53db-43c6-bf03-30b776995559","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:50.257Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7ff9a888-53db-43c6-bf03-30b776995559","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:50.257Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7ff9a888-53db-43c6-bf03-30b776995559","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:50.257Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7ff9a888-53db-43c6-bf03-30b776995559"}
{"level":"info","ts":"2026-09-18T09:53:50.257Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7ff9a888-53db-43c6-bf03-30b776995559","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:50.257Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7ff9a888-53db-43c6-bf03-30b776995559","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:50.262Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"24b12896-cdc0-445b-9b08-9ee20fb0b7ac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:50.262Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"24b12896-cdc0-445b-9b08-9ee20fb0b7ac","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:50.262Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"24b12896-cdc0-445b-9b08-9ee20fb0b7ac","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:50.262Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"24b12896-cdc0-445b-9b08-9ee20fb0b7ac"}
{"level":"info","ts":"2026-09-18T09:53:50.262Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"24b12896-cdc0-445b-9b08-9ee20fb0b7ac","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:50.262Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"24b12896-cdc0-445b-9b08-9ee20fb0b7ac","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:50.263Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7b5864b9-c6a5-423b-9145-597932836c79","name":"counter-fs5bg-workers"}
{"level":"info","ts":"2026-09-18T09:53:50.263Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7b5864b9-c6a5-423b-9145-597932836c79","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:52.235Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06942aac-6e60-43ab-95ea-d113b6b836d7"}
{"level":"info","ts":"2026-09-18T09:53:52.235Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06942aac-6e60-43ab-95ea-d113b6b836d7"}
{"level":"info","ts":"2026-09-18T09:53:52.235Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06942aac-6e60-43ab-95ea-d113b6b836d7"}
{"level":"info","ts":"2026-09-18T09:53:52.235Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06942aac-6e60-43ab-95ea-d113b6b836d7","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:53:52.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06942aac-6e60-43ab-95ea-d113b6b836d7","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:53:52.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06942aac-6e60-43ab-95ea-d113b6b836d7","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:52.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06942aac-6e60-43ab-95ea-d113b6b836d7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:53:52.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06942aac-6e60-43ab-95ea-d113b6b836d7","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:53:52.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06942aac-6e60-43ab-95ea-d113b6b836d7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:52.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06942aac-6e60-43ab-95ea-d113b6b836d7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:53:52.235Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06942aac-6e60-43ab-95ea-d113b6b836d7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:53:52.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06942aac-6e60-43ab-95ea-d113b6b836d7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:53:52.235Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06942aac-6e60-43ab-95ea-d113b6b836d7","podName":"counter-fs5bg-small-group-worker-nrzl6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:52.235Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06942aac-6e60-43ab-95ea-d113b6b836d7","podName":"counter-fs5bg-small-group-worker-nrzl6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:53:52.235Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06942aac-6e60-43ab-95ea-d113b6b836d7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:53:52.235Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"06942aac-6e60-43ab-95ea-d113b6b836d7","seconds":300}
{"level":"info","ts":"2026-09-18T09:53:53.263Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7a0946a6-653f-41b5-96c6-5e6d8aad85e0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:53.263Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7a0946a6-653f-41b5-96c6-5e6d8aad85e0","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:53.263Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7a0946a6-653f-41b5-96c6-5e6d8aad85e0","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:53.263Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7a0946a6-653f-41b5-96c6-5e6d8aad85e0"}
{"level":"info","ts":"2026-09-18T09:53:53.263Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7a0946a6-653f-41b5-96c6-5e6d8aad85e0","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:53.263Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7a0946a6-653f-41b5-96c6-5e6d8aad85e0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:56.264Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"51a5bbd6-60fb-4dcc-8db5-3803496823df","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:56.264Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"51a5bbd6-60fb-4dcc-8db5-3803496823df","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:56.264Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"51a5bbd6-60fb-4dcc-8db5-3803496823df","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:56.264Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"51a5bbd6-60fb-4dcc-8db5-3803496823df"}
{"level":"info","ts":"2026-09-18T09:53:56.264Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"51a5bbd6-60fb-4dcc-8db5-3803496823df","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:56.264Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"51a5bbd6-60fb-4dcc-8db5-3803496823df","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:53:56.391Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"3c703b10-52bd-4fd8-a435-c2d4f729bc11","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:56.391Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"3c703b10-52bd-4fd8-a435-c2d4f729bc11","pod":"counter-fs5bg-head-lbrhk","ip":"10.133.0.38"}
{"level":"info","ts":"2026-09-18T09:53:56.391Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"3c703b10-52bd-4fd8-a435-c2d4f729bc11","pod":"counter-fs5bg-small-group-worker-nrzl6","ip":"10.132.0.37"}
{"level":"info","ts":"2026-09-18T09:53:56.391Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"3c703b10-52bd-4fd8-a435-c2d4f729bc11","rayCluster":"counter-fs5bg","podIPs":["10.133.0.38","10.132.0.37"]}
{"level":"info","ts":"2026-09-18T09:53:59.265Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fa7b53e1-f14b-4558-a647-166f29a0425c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:53:59.265Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fa7b53e1-f14b-4558-a647-166f29a0425c","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:53:59.265Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fa7b53e1-f14b-4558-a647-166f29a0425c","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:53:59.265Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fa7b53e1-f14b-4558-a647-166f29a0425c"}
{"level":"info","ts":"2026-09-18T09:53:59.265Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fa7b53e1-f14b-4558-a647-166f29a0425c","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:53:59.265Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fa7b53e1-f14b-4558-a647-166f29a0425c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:02.266Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"961468e3-d603-4531-82c4-65fea1ba1e33","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:02.266Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"961468e3-d603-4531-82c4-65fea1ba1e33","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:02.266Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"961468e3-d603-4531-82c4-65fea1ba1e33","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:02.266Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"961468e3-d603-4531-82c4-65fea1ba1e33"}
{"level":"info","ts":"2026-09-18T09:54:02.266Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"961468e3-d603-4531-82c4-65fea1ba1e33","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:02.266Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"961468e3-d603-4531-82c4-65fea1ba1e33","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:05.266Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7fd5cdd4-ae92-46e3-88a7-be1c8d32bd98","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:05.266Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7fd5cdd4-ae92-46e3-88a7-be1c8d32bd98","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:05.267Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7fd5cdd4-ae92-46e3-88a7-be1c8d32bd98","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:05.267Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7fd5cdd4-ae92-46e3-88a7-be1c8d32bd98"}
{"level":"info","ts":"2026-09-18T09:54:05.267Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7fd5cdd4-ae92-46e3-88a7-be1c8d32bd98","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:05.267Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7fd5cdd4-ae92-46e3-88a7-be1c8d32bd98","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:08.268Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ea3a0362-07da-4fed-820c-7ab2fd753c71","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:08.268Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ea3a0362-07da-4fed-820c-7ab2fd753c71","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:08.268Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ea3a0362-07da-4fed-820c-7ab2fd753c71","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:08.268Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ea3a0362-07da-4fed-820c-7ab2fd753c71"}
{"level":"info","ts":"2026-09-18T09:54:08.268Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ea3a0362-07da-4fed-820c-7ab2fd753c71","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:08.268Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ea3a0362-07da-4fed-820c-7ab2fd753c71","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:11.269Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fdf5659d-d4e5-4a31-956c-9f1f770068d3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:11.269Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fdf5659d-d4e5-4a31-956c-9f1f770068d3","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:11.269Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fdf5659d-d4e5-4a31-956c-9f1f770068d3","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:11.269Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fdf5659d-d4e5-4a31-956c-9f1f770068d3"}
{"level":"info","ts":"2026-09-18T09:54:11.269Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fdf5659d-d4e5-4a31-956c-9f1f770068d3","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:11.269Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"fdf5659d-d4e5-4a31-956c-9f1f770068d3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:14.269Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f5acdc8d-60eb-41cf-9b1f-de0b4ff27365","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:14.269Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f5acdc8d-60eb-41cf-9b1f-de0b4ff27365","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:14.269Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f5acdc8d-60eb-41cf-9b1f-de0b4ff27365","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:14.269Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f5acdc8d-60eb-41cf-9b1f-de0b4ff27365"}
{"level":"info","ts":"2026-09-18T09:54:14.269Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f5acdc8d-60eb-41cf-9b1f-de0b4ff27365","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:14.269Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f5acdc8d-60eb-41cf-9b1f-de0b4ff27365","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:17.270Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c01ee36c-e613-4bed-9932-da5fa134f230","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:17.270Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c01ee36c-e613-4bed-9932-da5fa134f230","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:17.270Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c01ee36c-e613-4bed-9932-da5fa134f230","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:17.270Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c01ee36c-e613-4bed-9932-da5fa134f230"}
{"level":"info","ts":"2026-09-18T09:54:17.270Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c01ee36c-e613-4bed-9932-da5fa134f230","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:17.270Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"c01ee36c-e613-4bed-9932-da5fa134f230","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:20.271Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"96e3c9d5-5b51-4110-9bc1-d95883f5cea2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:20.271Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"96e3c9d5-5b51-4110-9bc1-d95883f5cea2","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:20.271Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"96e3c9d5-5b51-4110-9bc1-d95883f5cea2","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:20.272Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"96e3c9d5-5b51-4110-9bc1-d95883f5cea2"}
{"level":"info","ts":"2026-09-18T09:54:20.272Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"96e3c9d5-5b51-4110-9bc1-d95883f5cea2","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:20.272Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"96e3c9d5-5b51-4110-9bc1-d95883f5cea2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:23.273Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"6865facf-8dca-4bfd-aa55-333ec7b2d3d9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:23.273Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"6865facf-8dca-4bfd-aa55-333ec7b2d3d9","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:23.273Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"6865facf-8dca-4bfd-aa55-333ec7b2d3d9","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:23.273Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"6865facf-8dca-4bfd-aa55-333ec7b2d3d9"}
{"level":"info","ts":"2026-09-18T09:54:23.273Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"6865facf-8dca-4bfd-aa55-333ec7b2d3d9","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:23.273Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"6865facf-8dca-4bfd-aa55-333ec7b2d3d9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:26.274Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"0f30d887-83fa-48bf-827d-b099ad5a1bd0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:26.274Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"0f30d887-83fa-48bf-827d-b099ad5a1bd0","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:26.274Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"0f30d887-83fa-48bf-827d-b099ad5a1bd0","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:26.274Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"0f30d887-83fa-48bf-827d-b099ad5a1bd0"}
{"level":"info","ts":"2026-09-18T09:54:26.274Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"0f30d887-83fa-48bf-827d-b099ad5a1bd0","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:26.274Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"0f30d887-83fa-48bf-827d-b099ad5a1bd0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:29.275Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"6460c830-b87f-454f-96b9-8b38d46c4e3b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:29.275Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"6460c830-b87f-454f-96b9-8b38d46c4e3b","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:29.275Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"6460c830-b87f-454f-96b9-8b38d46c4e3b","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:29.275Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"6460c830-b87f-454f-96b9-8b38d46c4e3b"}
{"level":"info","ts":"2026-09-18T09:54:29.275Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"6460c830-b87f-454f-96b9-8b38d46c4e3b","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:29.275Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"6460c830-b87f-454f-96b9-8b38d46c4e3b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:32.276Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a6e1ff97-c115-48a8-8773-da7bfb7d9071","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:32.276Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a6e1ff97-c115-48a8-8773-da7bfb7d9071","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:32.276Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a6e1ff97-c115-48a8-8773-da7bfb7d9071","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:32.276Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a6e1ff97-c115-48a8-8773-da7bfb7d9071"}
{"level":"info","ts":"2026-09-18T09:54:32.276Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a6e1ff97-c115-48a8-8773-da7bfb7d9071","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:32.276Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a6e1ff97-c115-48a8-8773-da7bfb7d9071","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:35.277Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3fda4658-923c-48bb-ab23-7005787bd544","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:35.277Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3fda4658-923c-48bb-ab23-7005787bd544","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:35.277Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3fda4658-923c-48bb-ab23-7005787bd544","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:35.277Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3fda4658-923c-48bb-ab23-7005787bd544"}
{"level":"info","ts":"2026-09-18T09:54:35.277Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3fda4658-923c-48bb-ab23-7005787bd544","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:35.277Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3fda4658-923c-48bb-ab23-7005787bd544","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:38.278Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e9680818-aba9-4161-a007-d8004eb1df9a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:38.278Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e9680818-aba9-4161-a007-d8004eb1df9a","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:38.278Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e9680818-aba9-4161-a007-d8004eb1df9a","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:38.278Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e9680818-aba9-4161-a007-d8004eb1df9a"}
{"level":"info","ts":"2026-09-18T09:54:38.278Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e9680818-aba9-4161-a007-d8004eb1df9a","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:38.278Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e9680818-aba9-4161-a007-d8004eb1df9a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:41.279Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"b3b81ad0-f245-46c3-9c24-5b2c2bb94813","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:41.279Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"b3b81ad0-f245-46c3-9c24-5b2c2bb94813","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:41.279Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"b3b81ad0-f245-46c3-9c24-5b2c2bb94813","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:41.279Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"b3b81ad0-f245-46c3-9c24-5b2c2bb94813"}
{"level":"info","ts":"2026-09-18T09:54:41.279Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"b3b81ad0-f245-46c3-9c24-5b2c2bb94813","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:41.279Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"b3b81ad0-f245-46c3-9c24-5b2c2bb94813","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:44.281Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"acc6876f-80c7-48da-bef2-afbd6110aacf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:44.281Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"acc6876f-80c7-48da-bef2-afbd6110aacf","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:44.281Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"acc6876f-80c7-48da-bef2-afbd6110aacf","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:44.281Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"acc6876f-80c7-48da-bef2-afbd6110aacf"}
{"level":"info","ts":"2026-09-18T09:54:44.281Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"acc6876f-80c7-48da-bef2-afbd6110aacf","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:44.281Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"acc6876f-80c7-48da-bef2-afbd6110aacf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:47.282Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3b182138-a2aa-48ee-8687-66cc117631ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:47.282Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3b182138-a2aa-48ee-8687-66cc117631ff","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:47.282Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3b182138-a2aa-48ee-8687-66cc117631ff","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:47.282Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3b182138-a2aa-48ee-8687-66cc117631ff"}
{"level":"info","ts":"2026-09-18T09:54:47.282Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3b182138-a2aa-48ee-8687-66cc117631ff","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:47.282Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3b182138-a2aa-48ee-8687-66cc117631ff","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:47.765Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c59460be-f5f0-4b89-9c35-bdfdc64e3659"}
{"level":"info","ts":"2026-09-18T09:54:47.765Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c59460be-f5f0-4b89-9c35-bdfdc64e3659"}
{"level":"info","ts":"2026-09-18T09:54:47.765Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c59460be-f5f0-4b89-9c35-bdfdc64e3659"}
{"level":"info","ts":"2026-09-18T09:54:47.765Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c59460be-f5f0-4b89-9c35-bdfdc64e3659","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:54:47.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c59460be-f5f0-4b89-9c35-bdfdc64e3659","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:54:47.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c59460be-f5f0-4b89-9c35-bdfdc64e3659","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:54:47.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c59460be-f5f0-4b89-9c35-bdfdc64e3659","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:54:47.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c59460be-f5f0-4b89-9c35-bdfdc64e3659","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:54:47.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c59460be-f5f0-4b89-9c35-bdfdc64e3659","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:54:47.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c59460be-f5f0-4b89-9c35-bdfdc64e3659","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:54:47.766Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c59460be-f5f0-4b89-9c35-bdfdc64e3659","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:54:47.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c59460be-f5f0-4b89-9c35-bdfdc64e3659","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:54:47.766Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c59460be-f5f0-4b89-9c35-bdfdc64e3659","podName":"counter-fs5bg-small-group-worker-nrzl6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:54:47.766Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c59460be-f5f0-4b89-9c35-bdfdc64e3659","podName":"counter-fs5bg-small-group-worker-nrzl6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:54:47.766Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c59460be-f5f0-4b89-9c35-bdfdc64e3659","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:54:47.766Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"c59460be-f5f0-4b89-9c35-bdfdc64e3659","seconds":300}
{"level":"info","ts":"2026-09-18T09:54:48.777Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e3f77608-783c-440c-b74f-4fb522f414c6"}
{"level":"info","ts":"2026-09-18T09:54:48.777Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e3f77608-783c-440c-b74f-4fb522f414c6"}
{"level":"info","ts":"2026-09-18T09:54:48.777Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e3f77608-783c-440c-b74f-4fb522f414c6"}
{"level":"info","ts":"2026-09-18T09:54:48.777Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e3f77608-783c-440c-b74f-4fb522f414c6","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:54:48.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e3f77608-783c-440c-b74f-4fb522f414c6","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:54:48.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e3f77608-783c-440c-b74f-4fb522f414c6","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:54:48.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e3f77608-783c-440c-b74f-4fb522f414c6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:54:48.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e3f77608-783c-440c-b74f-4fb522f414c6","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:54:48.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e3f77608-783c-440c-b74f-4fb522f414c6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:54:48.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e3f77608-783c-440c-b74f-4fb522f414c6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:54:48.778Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e3f77608-783c-440c-b74f-4fb522f414c6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:54:48.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e3f77608-783c-440c-b74f-4fb522f414c6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:54:48.778Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e3f77608-783c-440c-b74f-4fb522f414c6","podName":"counter-fs5bg-small-group-worker-nrzl6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:54:48.778Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e3f77608-783c-440c-b74f-4fb522f414c6","podName":"counter-fs5bg-small-group-worker-nrzl6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:54:48.778Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e3f77608-783c-440c-b74f-4fb522f414c6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:54:48.778Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"e3f77608-783c-440c-b74f-4fb522f414c6","seconds":300}
{"level":"info","ts":"2026-09-18T09:54:50.283Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"120d334a-9f30-413e-ba41-82df14be3a39","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:50.283Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"120d334a-9f30-413e-ba41-82df14be3a39","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:50.283Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"120d334a-9f30-413e-ba41-82df14be3a39","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:50.283Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"120d334a-9f30-413e-ba41-82df14be3a39"}
{"level":"info","ts":"2026-09-18T09:54:50.283Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"120d334a-9f30-413e-ba41-82df14be3a39","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:50.283Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"120d334a-9f30-413e-ba41-82df14be3a39","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:53.284Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f2336d7d-c001-4bb5-9209-e6b98531a4d9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:53.284Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f2336d7d-c001-4bb5-9209-e6b98531a4d9","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:53.284Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f2336d7d-c001-4bb5-9209-e6b98531a4d9","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:53.284Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f2336d7d-c001-4bb5-9209-e6b98531a4d9"}
{"level":"info","ts":"2026-09-18T09:54:53.284Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f2336d7d-c001-4bb5-9209-e6b98531a4d9","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:53.284Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"f2336d7d-c001-4bb5-9209-e6b98531a4d9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:56.284Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"175e5428-30e1-4456-b135-9a4b4d10ae94","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:56.284Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"175e5428-30e1-4456-b135-9a4b4d10ae94","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:56.285Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"175e5428-30e1-4456-b135-9a4b4d10ae94","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:56.285Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"175e5428-30e1-4456-b135-9a4b4d10ae94"}
{"level":"info","ts":"2026-09-18T09:54:56.285Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"175e5428-30e1-4456-b135-9a4b4d10ae94","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:56.285Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"175e5428-30e1-4456-b135-9a4b4d10ae94","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:56.392Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"8d20713f-0302-4acd-8ba0-f3df3785da9a","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:56.392Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"8d20713f-0302-4acd-8ba0-f3df3785da9a","pod":"counter-fs5bg-head-lbrhk","ip":"10.133.0.38"}
{"level":"info","ts":"2026-09-18T09:54:56.392Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"8d20713f-0302-4acd-8ba0-f3df3785da9a","pod":"counter-fs5bg-small-group-worker-nrzl6","ip":"10.132.0.37"}
{"level":"info","ts":"2026-09-18T09:54:56.392Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"8d20713f-0302-4acd-8ba0-f3df3785da9a","rayCluster":"counter-fs5bg","podIPs":["10.133.0.38","10.132.0.37"]}
{"level":"info","ts":"2026-09-18T09:54:57.818Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6b36bfa9-5c90-4a38-a841-78b135ce4306"}
{"level":"info","ts":"2026-09-18T09:54:57.818Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6b36bfa9-5c90-4a38-a841-78b135ce4306"}
{"level":"info","ts":"2026-09-18T09:54:57.818Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6b36bfa9-5c90-4a38-a841-78b135ce4306"}
{"level":"info","ts":"2026-09-18T09:54:57.818Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6b36bfa9-5c90-4a38-a841-78b135ce4306","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:54:57.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6b36bfa9-5c90-4a38-a841-78b135ce4306","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:54:57.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6b36bfa9-5c90-4a38-a841-78b135ce4306","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:54:57.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6b36bfa9-5c90-4a38-a841-78b135ce4306","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:54:57.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6b36bfa9-5c90-4a38-a841-78b135ce4306","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:54:57.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6b36bfa9-5c90-4a38-a841-78b135ce4306","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:54:57.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6b36bfa9-5c90-4a38-a841-78b135ce4306","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:54:57.818Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6b36bfa9-5c90-4a38-a841-78b135ce4306","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:54:57.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6b36bfa9-5c90-4a38-a841-78b135ce4306","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:54:57.818Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6b36bfa9-5c90-4a38-a841-78b135ce4306","podName":"counter-fs5bg-small-group-worker-nrzl6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:54:57.819Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6b36bfa9-5c90-4a38-a841-78b135ce4306","podName":"counter-fs5bg-small-group-worker-nrzl6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:54:57.819Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6b36bfa9-5c90-4a38-a841-78b135ce4306","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:54:57.819Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"6b36bfa9-5c90-4a38-a841-78b135ce4306","seconds":300}
{"level":"info","ts":"2026-09-18T09:54:58.834Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"1cb1474e-1f24-4540-bd72-1fd33e246c18"}
{"level":"info","ts":"2026-09-18T09:54:58.834Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"1cb1474e-1f24-4540-bd72-1fd33e246c18"}
{"level":"info","ts":"2026-09-18T09:54:58.834Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"1cb1474e-1f24-4540-bd72-1fd33e246c18"}
{"level":"info","ts":"2026-09-18T09:54:58.834Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"1cb1474e-1f24-4540-bd72-1fd33e246c18","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:54:58.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"1cb1474e-1f24-4540-bd72-1fd33e246c18","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:54:58.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"1cb1474e-1f24-4540-bd72-1fd33e246c18","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:54:58.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"1cb1474e-1f24-4540-bd72-1fd33e246c18","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:54:58.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"1cb1474e-1f24-4540-bd72-1fd33e246c18","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:54:58.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"1cb1474e-1f24-4540-bd72-1fd33e246c18","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:54:58.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"1cb1474e-1f24-4540-bd72-1fd33e246c18","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:54:58.834Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"1cb1474e-1f24-4540-bd72-1fd33e246c18","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:54:58.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"1cb1474e-1f24-4540-bd72-1fd33e246c18","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:54:58.835Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"1cb1474e-1f24-4540-bd72-1fd33e246c18","podName":"counter-fs5bg-small-group-worker-nrzl6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:53:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:54:58.835Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"1cb1474e-1f24-4540-bd72-1fd33e246c18","podName":"counter-fs5bg-small-group-worker-nrzl6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:53:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:54:58.835Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"1cb1474e-1f24-4540-bd72-1fd33e246c18","name":"counter-fs5bg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:54:58.847Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"2a64d3f5-7c28-4375-88f3-6660bd7daa98","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:58.847Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"2a64d3f5-7c28-4375-88f3-6660bd7daa98","pod":"counter-fs5bg-head-lbrhk","ip":"10.133.0.38"}
{"level":"info","ts":"2026-09-18T09:54:58.847Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"2a64d3f5-7c28-4375-88f3-6660bd7daa98","pod":"counter-fs5bg-small-group-worker-nrzl6","ip":"10.132.0.37"}
{"level":"info","ts":"2026-09-18T09:54:58.847Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"2a64d3f5-7c28-4375-88f3-6660bd7daa98","rayCluster":"counter-fs5bg","podIPs":["10.133.0.38","10.132.0.37"]}
{"level":"info","ts":"2026-09-18T09:54:58.847Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2c30dd87-5454-47ec-ac7b-e70e06b722cb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:58.847Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2c30dd87-5454-47ec-ac7b-e70e06b722cb","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:58.847Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2c30dd87-5454-47ec-ac7b-e70e06b722cb","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:58.847Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"699fa3b7-d976-4e5a-b3d8-78d2daa129ba","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:58.847Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2c30dd87-5454-47ec-ac7b-e70e06b722cb"}
{"level":"info","ts":"2026-09-18T09:54:58.847Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2c30dd87-5454-47ec-ac7b-e70e06b722cb","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:58.847Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2c30dd87-5454-47ec-ac7b-e70e06b722cb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:53:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:58.847Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2c30dd87-5454-47ec-ac7b-e70e06b722cb","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:54:58.860Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"566b04e5-ea11-45e6-a081-28eb6d9f4078","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:58.860Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"566b04e5-ea11-45e6-a081-28eb6d9f4078","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:58.860Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"566b04e5-ea11-45e6-a081-28eb6d9f4078","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:58.861Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"566b04e5-ea11-45e6-a081-28eb6d9f4078"}
{"level":"info","ts":"2026-09-18T09:54:58.861Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"566b04e5-ea11-45e6-a081-28eb6d9f4078","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:58.861Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"566b04e5-ea11-45e6-a081-28eb6d9f4078","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:54:58.866Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"699fa3b7-d976-4e5a-b3d8-78d2daa129ba","name":"counter-fs5bg-head"}
{"level":"info","ts":"2026-09-18T09:54:58.875Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"699fa3b7-d976-4e5a-b3d8-78d2daa129ba","name":"counter-fs5bg-workers"}
{"level":"info","ts":"2026-09-18T09:54:58.875Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"699fa3b7-d976-4e5a-b3d8-78d2daa129ba","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:59.285Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2897b893-d2a1-4b56-973a-b82d112315c4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:54:59.285Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2897b893-d2a1-4b56-973a-b82d112315c4","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:54:59.285Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2897b893-d2a1-4b56-973a-b82d112315c4","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:54:59.285Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2897b893-d2a1-4b56-973a-b82d112315c4"}
{"level":"info","ts":"2026-09-18T09:54:59.286Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2897b893-d2a1-4b56-973a-b82d112315c4","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:54:59.286Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2897b893-d2a1-4b56-973a-b82d112315c4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:55:00.847Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2067258c-9926-4c0a-9eed-b7f89f1ffca0"}
{"level":"info","ts":"2026-09-18T09:55:00.847Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2067258c-9926-4c0a-9eed-b7f89f1ffca0"}
{"level":"info","ts":"2026-09-18T09:55:00.847Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2067258c-9926-4c0a-9eed-b7f89f1ffca0"}
{"level":"info","ts":"2026-09-18T09:55:00.847Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2067258c-9926-4c0a-9eed-b7f89f1ffca0","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:55:00.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2067258c-9926-4c0a-9eed-b7f89f1ffca0","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:55:00.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2067258c-9926-4c0a-9eed-b7f89f1ffca0","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:55:00.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2067258c-9926-4c0a-9eed-b7f89f1ffca0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:55:00.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2067258c-9926-4c0a-9eed-b7f89f1ffca0","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:55:00.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2067258c-9926-4c0a-9eed-b7f89f1ffca0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:55:00.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2067258c-9926-4c0a-9eed-b7f89f1ffca0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:55:00.847Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2067258c-9926-4c0a-9eed-b7f89f1ffca0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:55:00.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2067258c-9926-4c0a-9eed-b7f89f1ffca0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:55:00.847Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2067258c-9926-4c0a-9eed-b7f89f1ffca0","podName":"counter-fs5bg-small-group-worker-nrzl6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:53:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:55:00.847Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2067258c-9926-4c0a-9eed-b7f89f1ffca0","podName":"counter-fs5bg-small-group-worker-nrzl6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:53:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:55:00.847Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2067258c-9926-4c0a-9eed-b7f89f1ffca0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:55:00.847Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2067258c-9926-4c0a-9eed-b7f89f1ffca0","seconds":300}
{"level":"info","ts":"2026-09-18T09:55:02.287Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a1c57c8a-df9b-4658-8490-ca16f55110af","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:02.287Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a1c57c8a-df9b-4658-8490-ca16f55110af","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:02.287Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a1c57c8a-df9b-4658-8490-ca16f55110af","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:02.287Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a1c57c8a-df9b-4658-8490-ca16f55110af"}
{"level":"info","ts":"2026-09-18T09:55:02.287Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a1c57c8a-df9b-4658-8490-ca16f55110af","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:55:02.287Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a1c57c8a-df9b-4658-8490-ca16f55110af","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:55:05.288Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e0326f56-a05e-4d3b-8840-012113e2f451","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:05.288Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e0326f56-a05e-4d3b-8840-012113e2f451","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:05.288Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e0326f56-a05e-4d3b-8840-012113e2f451","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:05.288Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e0326f56-a05e-4d3b-8840-012113e2f451"}
{"level":"info","ts":"2026-09-18T09:55:05.288Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e0326f56-a05e-4d3b-8840-012113e2f451","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:55:05.288Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"e0326f56-a05e-4d3b-8840-012113e2f451","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:55:08.288Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"b5889681-6d96-4848-baae-02a9dc58c4e2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:08.288Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"b5889681-6d96-4848-baae-02a9dc58c4e2","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:08.288Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"b5889681-6d96-4848-baae-02a9dc58c4e2","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:08.288Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"b5889681-6d96-4848-baae-02a9dc58c4e2"}
{"level":"info","ts":"2026-09-18T09:55:08.288Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"b5889681-6d96-4848-baae-02a9dc58c4e2","RayCluster":"counter-fs5bg","State":""}
{"level":"info","ts":"2026-09-18T09:55:08.288Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"b5889681-6d96-4848-baae-02a9dc58c4e2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:55:09.878Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"7a10b7c4-60d1-4c9e-a266-78644d7b902a"}
{"level":"info","ts":"2026-09-18T09:55:09.878Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"7a10b7c4-60d1-4c9e-a266-78644d7b902a"}
{"level":"info","ts":"2026-09-18T09:55:09.878Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"7a10b7c4-60d1-4c9e-a266-78644d7b902a"}
{"level":"info","ts":"2026-09-18T09:55:09.878Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"7a10b7c4-60d1-4c9e-a266-78644d7b902a","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:55:09.878Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"7a10b7c4-60d1-4c9e-a266-78644d7b902a","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:55:09.878Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"7a10b7c4-60d1-4c9e-a266-78644d7b902a","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:55:09.878Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"7a10b7c4-60d1-4c9e-a266-78644d7b902a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:55:09.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"7a10b7c4-60d1-4c9e-a266-78644d7b902a","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:55:09.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"7a10b7c4-60d1-4c9e-a266-78644d7b902a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:55:09.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"7a10b7c4-60d1-4c9e-a266-78644d7b902a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:55:09.879Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"7a10b7c4-60d1-4c9e-a266-78644d7b902a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:55:09.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"7a10b7c4-60d1-4c9e-a266-78644d7b902a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:55:09.879Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"7a10b7c4-60d1-4c9e-a266-78644d7b902a","name":"counter-fs5bg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:55:09.894Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"d7bb0d90-96b8-499d-b16b-40bfcc46971e","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:09.895Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7e3ab91b-9d74-45f3-b4fb-418cd0abe8a5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:09.895Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7e3ab91b-9d74-45f3-b4fb-418cd0abe8a5","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:09.895Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7e3ab91b-9d74-45f3-b4fb-418cd0abe8a5","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:09.895Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"d7bb0d90-96b8-499d-b16b-40bfcc46971e","pod":"counter-fs5bg-head-lbrhk","ip":"10.133.0.38"}
{"level":"info","ts":"2026-09-18T09:55:09.895Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"d7bb0d90-96b8-499d-b16b-40bfcc46971e","pod":"counter-fs5bg-small-group-worker-nrzl6","ip":"10.132.0.37"}
{"level":"info","ts":"2026-09-18T09:55:09.895Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"d7bb0d90-96b8-499d-b16b-40bfcc46971e","rayCluster":"counter-fs5bg","podIPs":["10.133.0.38","10.132.0.37"]}
{"level":"info","ts":"2026-09-18T09:55:09.895Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7e3ab91b-9d74-45f3-b4fb-418cd0abe8a5"}
{"level":"info","ts":"2026-09-18T09:55:09.895Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"441fa83a-daa7-4dd1-918e-ba722d98b2c8","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:09.910Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7e3ab91b-9d74-45f3-b4fb-418cd0abe8a5","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-09-18T09:55:09.910Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7e3ab91b-9d74-45f3-b4fb-418cd0abe8a5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:55:09.910Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7e3ab91b-9d74-45f3-b4fb-418cd0abe8a5","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:55:09.913Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"441fa83a-daa7-4dd1-918e-ba722d98b2c8","name":"counter-fs5bg-head"}
{"level":"info","ts":"2026-09-18T09:55:09.923Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8beb0844-37ed-45d7-ac32-9ce8138fda30","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:09.923Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8beb0844-37ed-45d7-ac32-9ce8138fda30","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:09.923Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8beb0844-37ed-45d7-ac32-9ce8138fda30","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:09.923Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8beb0844-37ed-45d7-ac32-9ce8138fda30"}
{"level":"info","ts":"2026-09-18T09:55:09.923Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8beb0844-37ed-45d7-ac32-9ce8138fda30","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-09-18T09:55:09.923Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8beb0844-37ed-45d7-ac32-9ce8138fda30","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:55:09.923Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8beb0844-37ed-45d7-ac32-9ce8138fda30","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:55:09.926Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"441fa83a-daa7-4dd1-918e-ba722d98b2c8","name":"counter-fs5bg-workers"}
{"level":"info","ts":"2026-09-18T09:55:09.926Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"441fa83a-daa7-4dd1-918e-ba722d98b2c8","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:09.933Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8beb0844-37ed-45d7-ac32-9ce8138fda30","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T09:55:09.933Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8beb0844-37ed-45d7-ac32-9ce8138fda30"}
{"level":"error","ts":"2026-09-18T09:55:09.933Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8beb0844-37ed-45d7-ac32-9ce8138fda30","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:09.934Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8f1001e2-c870-4164-9a4c-afd097496f76","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:09.934Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8f1001e2-c870-4164-9a4c-afd097496f76","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:09.934Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8f1001e2-c870-4164-9a4c-afd097496f76","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:09.934Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8f1001e2-c870-4164-9a4c-afd097496f76"}
{"level":"info","ts":"2026-09-18T09:55:09.934Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8f1001e2-c870-4164-9a4c-afd097496f76","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:55:09.974Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8f1001e2-c870-4164-9a4c-afd097496f76","JobId":"counter-dpfrz","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:09.975Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8f1001e2-c870-4164-9a4c-afd097496f76"}
{"level":"error","ts":"2026-09-18T09:55:09.975Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8f1001e2-c870-4164-9a4c-afd097496f76","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:09.975Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"5e8793ba-2f37-4e24-a7f5-a71d85c1ded7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:09.975Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"5e8793ba-2f37-4e24-a7f5-a71d85c1ded7","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:09.975Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"5e8793ba-2f37-4e24-a7f5-a71d85c1ded7","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:09.975Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"5e8793ba-2f37-4e24-a7f5-a71d85c1ded7"}
{"level":"info","ts":"2026-09-18T09:55:09.975Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"5e8793ba-2f37-4e24-a7f5-a71d85c1ded7","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:55:09.979Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"5e8793ba-2f37-4e24-a7f5-a71d85c1ded7","JobId":"counter-dpfrz","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:09.979Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"5e8793ba-2f37-4e24-a7f5-a71d85c1ded7"}
{"level":"error","ts":"2026-09-18T09:55:09.979Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"5e8793ba-2f37-4e24-a7f5-a71d85c1ded7","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:09.985Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7c55f194-b11c-45fb-af57-680919bac744","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:09.985Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7c55f194-b11c-45fb-af57-680919bac744","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:09.985Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7c55f194-b11c-45fb-af57-680919bac744","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:09.985Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7c55f194-b11c-45fb-af57-680919bac744"}
{"level":"info","ts":"2026-09-18T09:55:09.985Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7c55f194-b11c-45fb-af57-680919bac744","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:55:09.989Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7c55f194-b11c-45fb-af57-680919bac744","JobId":"counter-dpfrz","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:09.989Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7c55f194-b11c-45fb-af57-680919bac744"}
{"level":"error","ts":"2026-09-18T09:55:09.989Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"7c55f194-b11c-45fb-af57-680919bac744","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:10.030Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2fc0aa67-ce16-46f6-9f3c-d74855a36c2e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:10.030Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2fc0aa67-ce16-46f6-9f3c-d74855a36c2e","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:10.030Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2fc0aa67-ce16-46f6-9f3c-d74855a36c2e","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:10.030Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2fc0aa67-ce16-46f6-9f3c-d74855a36c2e"}
{"level":"info","ts":"2026-09-18T09:55:10.030Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2fc0aa67-ce16-46f6-9f3c-d74855a36c2e","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:55:10.034Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2fc0aa67-ce16-46f6-9f3c-d74855a36c2e","JobId":"counter-dpfrz","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:10.034Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2fc0aa67-ce16-46f6-9f3c-d74855a36c2e"}
{"level":"error","ts":"2026-09-18T09:55:10.034Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"2fc0aa67-ce16-46f6-9f3c-d74855a36c2e","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:10.115Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3e453781-0786-4c1a-bdde-33df7d4f286f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:10.115Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3e453781-0786-4c1a-bdde-33df7d4f286f","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:10.115Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3e453781-0786-4c1a-bdde-33df7d4f286f","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:10.115Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3e453781-0786-4c1a-bdde-33df7d4f286f"}
{"level":"info","ts":"2026-09-18T09:55:10.115Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3e453781-0786-4c1a-bdde-33df7d4f286f","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:55:10.120Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3e453781-0786-4c1a-bdde-33df7d4f286f","JobId":"counter-dpfrz","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:10.120Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3e453781-0786-4c1a-bdde-33df7d4f286f"}
{"level":"error","ts":"2026-09-18T09:55:10.120Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"3e453781-0786-4c1a-bdde-33df7d4f286f","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:10.280Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"333c63b4-6b11-48f5-a5cf-3802eb321cb7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:10.280Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"333c63b4-6b11-48f5-a5cf-3802eb321cb7","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:10.281Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"333c63b4-6b11-48f5-a5cf-3802eb321cb7","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:10.281Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"333c63b4-6b11-48f5-a5cf-3802eb321cb7"}
{"level":"info","ts":"2026-09-18T09:55:10.281Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"333c63b4-6b11-48f5-a5cf-3802eb321cb7","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:55:10.286Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"333c63b4-6b11-48f5-a5cf-3802eb321cb7","JobId":"counter-dpfrz","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:10.286Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"333c63b4-6b11-48f5-a5cf-3802eb321cb7"}
{"level":"error","ts":"2026-09-18T09:55:10.286Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"333c63b4-6b11-48f5-a5cf-3802eb321cb7","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:10.607Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"88612b52-c075-4cf9-a3f9-3e663fe2500d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:10.607Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"88612b52-c075-4cf9-a3f9-3e663fe2500d","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:10.607Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"88612b52-c075-4cf9-a3f9-3e663fe2500d","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:10.607Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"88612b52-c075-4cf9-a3f9-3e663fe2500d"}
{"level":"info","ts":"2026-09-18T09:55:10.607Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"88612b52-c075-4cf9-a3f9-3e663fe2500d","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:55:10.612Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"88612b52-c075-4cf9-a3f9-3e663fe2500d","JobId":"counter-dpfrz","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:10.612Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"88612b52-c075-4cf9-a3f9-3e663fe2500d"}
{"level":"error","ts":"2026-09-18T09:55:10.612Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"88612b52-c075-4cf9-a3f9-3e663fe2500d","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:10.933Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ef3ff09d-85d3-4d92-828a-33932dee16ec","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:10.933Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ef3ff09d-85d3-4d92-828a-33932dee16ec","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:10.933Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ef3ff09d-85d3-4d92-828a-33932dee16ec","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:10.933Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ef3ff09d-85d3-4d92-828a-33932dee16ec"}
{"level":"info","ts":"2026-09-18T09:55:10.933Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ef3ff09d-85d3-4d92-828a-33932dee16ec","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:55:10.937Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ef3ff09d-85d3-4d92-828a-33932dee16ec","JobId":"counter-dpfrz","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:10.937Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ef3ff09d-85d3-4d92-828a-33932dee16ec"}
{"level":"error","ts":"2026-09-18T09:55:10.937Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ef3ff09d-85d3-4d92-828a-33932dee16ec","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:11.252Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"33aa563b-20f2-41a7-83f0-5bf44c4a6963","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:11.252Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"33aa563b-20f2-41a7-83f0-5bf44c4a6963","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:11.253Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"33aa563b-20f2-41a7-83f0-5bf44c4a6963","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:11.253Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"33aa563b-20f2-41a7-83f0-5bf44c4a6963"}
{"level":"info","ts":"2026-09-18T09:55:11.253Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"33aa563b-20f2-41a7-83f0-5bf44c4a6963","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:55:11.257Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"33aa563b-20f2-41a7-83f0-5bf44c4a6963","JobId":"counter-dpfrz","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:11.257Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"33aa563b-20f2-41a7-83f0-5bf44c4a6963"}
{"level":"error","ts":"2026-09-18T09:55:11.257Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"33aa563b-20f2-41a7-83f0-5bf44c4a6963","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:11.895Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2822bd73-0228-434a-a41a-b78e1123aa4b"}
{"level":"info","ts":"2026-09-18T09:55:11.895Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2822bd73-0228-434a-a41a-b78e1123aa4b"}
{"level":"info","ts":"2026-09-18T09:55:11.895Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2822bd73-0228-434a-a41a-b78e1123aa4b"}
{"level":"info","ts":"2026-09-18T09:55:11.895Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2822bd73-0228-434a-a41a-b78e1123aa4b","1 head service found":"counter-fs5bg-head-svc"}
{"level":"info","ts":"2026-09-18T09:55:11.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2822bd73-0228-434a-a41a-b78e1123aa4b","Found 1 head Pod":"counter-fs5bg-head-lbrhk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:55:11.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2822bd73-0228-434a-a41a-b78e1123aa4b","head Pod":"counter-fs5bg-head-lbrhk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-fs5bg-head-lbrhk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:55:11.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2822bd73-0228-434a-a41a-b78e1123aa4b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:55:11.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2822bd73-0228-434a-a41a-b78e1123aa4b","worker Pod":"counter-fs5bg-small-group-worker-nrzl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-fs5bg-small-group-worker-nrzl6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:55:11.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2822bd73-0228-434a-a41a-b78e1123aa4b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:55:11.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2822bd73-0228-434a-a41a-b78e1123aa4b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:55:11.895Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2822bd73-0228-434a-a41a-b78e1123aa4b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:55:11.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2822bd73-0228-434a-a41a-b78e1123aa4b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:55:11.896Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2822bd73-0228-434a-a41a-b78e1123aa4b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:55:11.896Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"2822bd73-0228-434a-a41a-b78e1123aa4b","seconds":300}
{"level":"info","ts":"2026-09-18T09:55:13.818Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8024b1b1-f2f9-456a-a852-064c4e34ec20","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:13.818Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8024b1b1-f2f9-456a-a852-064c4e34ec20","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:13.818Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8024b1b1-f2f9-456a-a852-064c4e34ec20","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:13.818Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8024b1b1-f2f9-456a-a852-064c4e34ec20"}
{"level":"info","ts":"2026-09-18T09:55:13.818Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8024b1b1-f2f9-456a-a852-064c4e34ec20","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:55:13.823Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8024b1b1-f2f9-456a-a852-064c4e34ec20","JobId":"counter-dpfrz","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:13.823Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8024b1b1-f2f9-456a-a852-064c4e34ec20"}
{"level":"error","ts":"2026-09-18T09:55:13.823Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"8024b1b1-f2f9-456a-a852-064c4e34ec20","error":"Job counter-dpfrz does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:18.943Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"47fc924e-2f3f-4374-94c6-3351569b5d5b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:18.943Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"47fc924e-2f3f-4374-94c6-3351569b5d5b","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:18.944Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"47fc924e-2f3f-4374-94c6-3351569b5d5b","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:18.944Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"47fc924e-2f3f-4374-94c6-3351569b5d5b"}
{"level":"info","ts":"2026-09-18T09:55:18.944Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"47fc924e-2f3f-4374-94c6-3351569b5d5b","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:55:18.949Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"47fc924e-2f3f-4374-94c6-3351569b5d5b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:54:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:55:18.949Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"47fc924e-2f3f-4374-94c6-3351569b5d5b","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:55:18.967Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"4d9ce975-b487-48fd-ac69-8763ad1494a2","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:18.967Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"4d9ce975-b487-48fd-ac69-8763ad1494a2","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:18.967Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"4d9ce975-b487-48fd-ac69-8763ad1494a2","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:18.967Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"4d9ce975-b487-48fd-ac69-8763ad1494a2"}
{"level":"info","ts":"2026-09-18T09:55:18.967Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"4d9ce975-b487-48fd-ac69-8763ad1494a2","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:55:18.970Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"4d9ce975-b487-48fd-ac69-8763ad1494a2","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:55:21.968Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"73080c4f-d841-41a4-a91b-3cc4eb5d17e9","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:21.968Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"73080c4f-d841-41a4-a91b-3cc4eb5d17e9","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:21.968Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"73080c4f-d841-41a4-a91b-3cc4eb5d17e9","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:21.968Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"73080c4f-d841-41a4-a91b-3cc4eb5d17e9"}
{"level":"info","ts":"2026-09-18T09:55:21.968Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"73080c4f-d841-41a4-a91b-3cc4eb5d17e9","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:55:21.971Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"73080c4f-d841-41a4-a91b-3cc4eb5d17e9","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z","endTime":"2026-09-18T09:55:21Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:55:21.971Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"73080c4f-d841-41a4-a91b-3cc4eb5d17e9","old JobStatus":"PENDING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:55:21.984Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"d98fa95c-c966-4165-9718-bb453aecbd30","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:21.984Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"d98fa95c-c966-4165-9718-bb453aecbd30","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:21.984Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"d98fa95c-c966-4165-9718-bb453aecbd30","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:21.984Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"d98fa95c-c966-4165-9718-bb453aecbd30"}
{"level":"info","ts":"2026-09-18T09:55:21.984Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"d98fa95c-c966-4165-9718-bb453aecbd30","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:55:21.987Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"d98fa95c-c966-4165-9718-bb453aecbd30","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z","endTime":"2026-09-18T09:55:21Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z","endTime":"2026-09-18T09:55:21Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:55:24.985Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a957194b-ad3a-44b3-b5bd-d49dc08b6149","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:24.985Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a957194b-ad3a-44b3-b5bd-d49dc08b6149","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:24.985Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a957194b-ad3a-44b3-b5bd-d49dc08b6149","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:24.985Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a957194b-ad3a-44b3-b5bd-d49dc08b6149"}
{"level":"info","ts":"2026-09-18T09:55:24.985Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a957194b-ad3a-44b3-b5bd-d49dc08b6149","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:55:24.988Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"a957194b-ad3a-44b3-b5bd-d49dc08b6149","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z","endTime":"2026-09-18T09:55:21Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z","endTime":"2026-09-18T09:55:21Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:55:27.989Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"61e4657f-6b40-4606-b2ef-f63b3280d8ea","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:27.989Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"61e4657f-6b40-4606-b2ef-f63b3280d8ea","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:27.989Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"61e4657f-6b40-4606-b2ef-f63b3280d8ea","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:27.989Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"61e4657f-6b40-4606-b2ef-f63b3280d8ea"}
{"level":"info","ts":"2026-09-18T09:55:27.989Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"61e4657f-6b40-4606-b2ef-f63b3280d8ea","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:55:27.993Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"61e4657f-6b40-4606-b2ef-f63b3280d8ea","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z","endTime":"2026-09-18T09:55:21Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z","endTime":"2026-09-18T09:55:21Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:55:28.965Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"cd8292ee-4e96-4232-b5fb-8a74027ace1c","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:28.965Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"cd8292ee-4e96-4232-b5fb-8a74027ace1c","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:28.965Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"cd8292ee-4e96-4232-b5fb-8a74027ace1c","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:28.965Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"cd8292ee-4e96-4232-b5fb-8a74027ace1c"}
{"level":"info","ts":"2026-09-18T09:55:28.965Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"cd8292ee-4e96-4232-b5fb-8a74027ace1c","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:55:28.969Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"cd8292ee-4e96-4232-b5fb-8a74027ace1c","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z","endTime":"2026-09-18T09:55:21Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z","endTime":"2026-09-18T09:55:21Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:55:29.972Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"1671a66e-a850-43e2-bac2-12bb7df16cd0","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:29.972Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"1671a66e-a850-43e2-bac2-12bb7df16cd0","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:29.972Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"1671a66e-a850-43e2-bac2-12bb7df16cd0","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:29.973Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"1671a66e-a850-43e2-bac2-12bb7df16cd0"}
{"level":"info","ts":"2026-09-18T09:55:29.973Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"1671a66e-a850-43e2-bac2-12bb7df16cd0","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:55:29.975Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"1671a66e-a850-43e2-bac2-12bb7df16cd0","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z","endTime":"2026-09-18T09:55:21Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z","endTime":"2026-09-18T09:55:21Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:55:29.991Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ec0d9531-2ac3-4a0d-be98-1155f774a62e","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:29.991Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ec0d9531-2ac3-4a0d-be98-1155f774a62e","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:29.991Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ec0d9531-2ac3-4a0d-be98-1155f774a62e","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:29.991Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ec0d9531-2ac3-4a0d-be98-1155f774a62e"}
{"level":"info","ts":"2026-09-18T09:55:29.991Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ec0d9531-2ac3-4a0d-be98-1155f774a62e","rayCluster":"counter-fs5bg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:55:29.993Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ec0d9531-2ac3-4a0d-be98-1155f774a62e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z","endTime":"2026-09-18T09:55:21Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T09:53:26Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:55:18Z","endTime":"2026-09-18T09:55:21Z"},"jobId":"counter-dpfrz","rayClusterName":"counter-fs5bg","dashboardURL":"counter-fs5bg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-09-18T09:53:26Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:55:09Z","stateTransitionTimes":{"ready":"2026-09-18T09:55:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"counter-fs5bg-head-lbrhk","serviceName":"counter-fs5bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:53:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:55:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:53:26Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:53:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:55:29.993Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ec0d9531-2ac3-4a0d-be98-1155f774a62e","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-09-18T09:55:30.007Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ff9b6cb5-2cfa-4509-801f-cd3ef821e533","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:30.007Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ff9b6cb5-2cfa-4509-801f-cd3ef821e533","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:55:29 +0000 UTC","Now":"2026-09-18T09:55:30.007Z","ShutdownTime":"2026-09-18T09:55:29.000Z"}
{"level":"info","ts":"2026-09-18T09:55:30.016Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ff9b6cb5-2cfa-4509-801f-cd3ef821e533","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:30.016Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ff9b6cb5-2cfa-4509-801f-cd3ef821e533","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T09:55:30.016Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"ff9b6cb5-2cfa-4509-801f-cd3ef821e533","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:30.017Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"reconcileID":"94bfb2fe-88d7-4cbb-8c7e-d1f9354f79da"}
{"level":"info","ts":"2026-09-18T09:55:30.017Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"8affcd12-3896-4b49-82b6-a7362c94201d","rayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:30.017Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7df20ba4-e27e-428b-b75f-ff385dc8119a","namespacedName":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:30.017Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7df20ba4-e27e-428b-b75f-ff385dc8119a","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:30.017Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7df20ba4-e27e-428b-b75f-ff385dc8119a","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:30.017Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"8affcd12-3896-4b49-82b6-a7362c94201d","pod":"counter-fs5bg-head-lbrhk","ip":"10.133.0.38"}
{"level":"info","ts":"2026-09-18T09:55:30.017Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"8affcd12-3896-4b49-82b6-a7362c94201d","pod":"counter-fs5bg-small-group-worker-nrzl6","ip":"10.132.0.37"}
{"level":"info","ts":"2026-09-18T09:55:30.017Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7df20ba4-e27e-428b-b75f-ff385dc8119a","namespace":"test-ns-ltx56","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T09:55:30.018Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"8affcd12-3896-4b49-82b6-a7362c94201d","rayCluster":"counter-fs5bg","podIPs":["10.133.0.38","10.132.0.37"]}
{"level":"info","ts":"2026-09-18T09:55:30.017Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"a918548f-c715-46ac-86fd-b2bf01504ccd"}
{"level":"info","ts":"2026-09-18T09:55:30.018Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"adaa4c6a-a644-48da-8987-0c7e671ad9a3","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:30.018Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"adaa4c6a-a644-48da-8987-0c7e671ad9a3","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:55:29 +0000 UTC","Now":"2026-09-18T09:55:30.018Z","ShutdownTime":"2026-09-18T09:55:29.000Z"}
{"level":"info","ts":"2026-09-18T09:55:30.018Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"adaa4c6a-a644-48da-8987-0c7e671ad9a3","RayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:30.018Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"adaa4c6a-a644-48da-8987-0c7e671ad9a3","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T09:55:30.018Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"adaa4c6a-a644-48da-8987-0c7e671ad9a3","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:30.022Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7df20ba4-e27e-428b-b75f-ff385dc8119a","name":"kuberay-gateway-access","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T09:55:30.028Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7df20ba4-e27e-428b-b75f-ff385dc8119a","name":"test-ns-ltx56-counter-fs5bg","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T09:55:30.032Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7df20ba4-e27e-428b-b75f-ff385dc8119a","configMap":"kube-rbac-proxy-config-counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:30.039Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7df20ba4-e27e-428b-b75f-ff385dc8119a","serviceAccount":"counter-fs5bg-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:55:30.069Z","logger":"raycluster-resource","msg":"default","name":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:30.069Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-fs5bg","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T09:55:30.079Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"7df20ba4-e27e-428b-b75f-ff385dc8119a","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:30.080Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"a1ce4acf-4e08-4ef8-8619-7658ca18c089","namespacedName":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:30.080Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"a1ce4acf-4e08-4ef8-8619-7658ca18c089","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:30.080Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"a1ce4acf-4e08-4ef8-8619-7658ca18c089","cluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:30.080Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"a1ce4acf-4e08-4ef8-8619-7658ca18c089","namespace":"test-ns-ltx56","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T09:55:30.080Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"a1ce4acf-4e08-4ef8-8619-7658ca18c089","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-09-18T09:55:30.080Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"70fc825e-553d-4725-87b6-d01b60c26685"}
{"level":"info","ts":"2026-09-18T09:55:30.080Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"94fbcc6e-f24b-495e-a551-681d93b8c420","namespace":"test-ns-ltx56","clusterName":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:30.080Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"10b6b228-3100-435e-b82f-069d0a93fb1b","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:30.080Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"10b6b228-3100-435e-b82f-069d0a93fb1b","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:55:29 +0000 UTC","Now":"2026-09-18T09:55:30.080Z","ShutdownTime":"2026-09-18T09:55:29.000Z"}
{"level":"info","ts":"2026-09-18T09:55:30.080Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"10b6b228-3100-435e-b82f-069d0a93fb1b","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:30.080Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"10b6b228-3100-435e-b82f-069d0a93fb1b","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T09:55:30.080Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"10b6b228-3100-435e-b82f-069d0a93fb1b","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-fs5bg"}
{"level":"error","ts":"2026-09-18T09:55:30.084Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"a1ce4acf-4e08-4ef8-8619-7658ca18c089","error":"rayclusters.ray.io \"counter-fs5bg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:196\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:137\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:30.084Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"a1ce4acf-4e08-4ef8-8619-7658ca18c089"}
{"level":"error","ts":"2026-09-18T09:55:30.084Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"a1ce4acf-4e08-4ef8-8619-7658ca18c089","error":"rayclusters.ray.io \"counter-fs5bg\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:30.084Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"bdfbc695-2f4b-4f0a-bedd-018f771372f0","namespacedName":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:30.084Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"bdfbc695-2f4b-4f0a-bedd-018f771372f0"}
{"level":"info","ts":"2026-09-18T09:55:30.084Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"bdfbc695-2f4b-4f0a-bedd-018f771372f0","namespace":"test-ns-ltx56","authClusters":0}
{"level":"info","ts":"2026-09-18T09:55:30.084Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"bdfbc695-2f4b-4f0a-bedd-018f771372f0","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T09:55:30.088Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"69632210-1055-47b3-a25e-fbf55b55101f"}
{"level":"info","ts":"2026-09-18T09:55:30.088Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"3db13be1-8655-4dd3-868a-35cc1cfa3409"}
{"level":"info","ts":"2026-09-18T09:55:30.089Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"380fd118-80ce-4ea1-903f-08a380370553","namespacedName":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:30.089Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"380fd118-80ce-4ea1-903f-08a380370553"}
{"level":"info","ts":"2026-09-18T09:55:30.089Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"380fd118-80ce-4ea1-903f-08a380370553","namespace":"test-ns-ltx56","authClusters":0}
{"level":"info","ts":"2026-09-18T09:55:30.089Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"380fd118-80ce-4ea1-903f-08a380370553","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T09:55:30.101Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"6bae6c42-b9ce-48de-aec0-a8c6759879bf","namespacedName":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:30.101Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"6bae6c42-b9ce-48de-aec0-a8c6759879bf"}
{"level":"info","ts":"2026-09-18T09:55:30.101Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"6bae6c42-b9ce-48de-aec0-a8c6759879bf","namespace":"test-ns-ltx56","authClusters":0}
{"level":"info","ts":"2026-09-18T09:55:30.101Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"6bae6c42-b9ce-48de-aec0-a8c6759879bf","namespace":"test-ns-ltx56"}
{"level":"error","ts":"2026-09-18T09:55:30.104Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"94fbcc6e-f24b-495e-a551-681d93b8c420","certificate":"ray-head-cert-counter-fs5bg","error":"certificates.cert-manager.io \"ray-head-cert-counter-fs5bg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:30.128Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"94fbcc6e-f24b-495e-a551-681d93b8c420","errorCount":1}
{"level":"info","ts":"2026-09-18T09:55:30.128Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"94fbcc6e-f24b-495e-a551-681d93b8c420"}
{"level":"error","ts":"2026-09-18T09:55:30.128Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"94fbcc6e-f24b-495e-a551-681d93b8c420","error":"certificates.cert-manager.io \"ray-head-cert-counter-fs5bg\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:55:30.134Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"4c0addf2-bc3a-4fae-8f62-e7210f32007c","namespace":"test-ns-ltx56","clusterName":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:30.134Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-fs5bg","reconcileID":"4c0addf2-bc3a-4fae-8f62-e7210f32007c","namespace":"test-ns-ltx56","clusterName":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:30.862Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"624da0b8-ae19-463c-8923-e189beede72e","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:30.862Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"624da0b8-ae19-463c-8923-e189beede72e","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:55:29 +0000 UTC","Now":"2026-09-18T09:55:30.862Z","ShutdownTime":"2026-09-18T09:55:29.000Z"}
{"level":"info","ts":"2026-09-18T09:55:30.862Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"624da0b8-ae19-463c-8923-e189beede72e","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:30.862Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"624da0b8-ae19-463c-8923-e189beede72e","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T09:55:30.862Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"624da0b8-ae19-463c-8923-e189beede72e","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:55:30.994Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"351758cb-b2e0-46df-b63a-90926573f955","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:55:30.994Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"351758cb-b2e0-46df-b63a-90926573f955","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 09:55:29 +0000 UTC","Now":"2026-09-18T09:55:30.994Z","ShutdownTime":"2026-09-18T09:55:29.000Z"}
{"level":"info","ts":"2026-09-18T09:55:30.994Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"351758cb-b2e0-46df-b63a-90926573f955","RayCluster":{"name":"counter-fs5bg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:55:30.994Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"351758cb-b2e0-46df-b63a-90926573f955","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T09:55:30.994Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"351758cb-b2e0-46df-b63a-90926573f955","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-fs5bg"}
{"level":"info","ts":"2026-09-18T09:56:00.873Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"4acfd40b-533b-4de6-829c-25bbb0d0633b","DeletionTimestamp":"2026-09-18 09:56:00 +0000 UTC"}
{"level":"info","ts":"2026-09-18T09:56:00.873Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"4acfd40b-533b-4de6-829c-25bbb0d0633b","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:56:00.892Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"66000bbf-f48c-454c-b23c-d95866406c1c","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:00.892Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"66000bbf-f48c-454c-b23c-d95866406c1c","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:56:00.904Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"66000bbf-f48c-454c-b23c-d95866406c1c"}
{"level":"info","ts":"2026-09-18T09:56:00.904Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"66000bbf-f48c-454c-b23c-d95866406c1c","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:56:00.904Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"66000bbf-f48c-454c-b23c-d95866406c1c","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:56:00.904Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"66000bbf-f48c-454c-b23c-d95866406c1c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:56:00.916Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"1ed56305-acde-4657-8bfc-fe94306eae8a"}
{"level":"info","ts":"2026-09-18T09:56:00.916Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"8ff20d7c-298e-4f04-bf40-096a7758b19a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:00.916Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"8ff20d7c-298e-4f04-bf40-096a7758b19a","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.916Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"8ff20d7c-298e-4f04-bf40-096a7758b19a","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:00.916Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"8ff20d7c-298e-4f04-bf40-096a7758b19a","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:00.923Z","logger":"raycluster-resource","msg":"default","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.923Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-qvndg","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"8ff20d7c-298e-4f04-bf40-096a7758b19a","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"fa7f664e-2706-4d40-81a8-d948c2af5d48","namespacedName":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"8ff20d7c-298e-4f04-bf40-096a7758b19a"}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"8ff20d7c-298e-4f04-bf40-096a7758b19a","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"fa7f664e-2706-4d40-81a8-d948c2af5d48","mode":"IntegratedOAuth","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"fa7f664e-2706-4d40-81a8-d948c2af5d48","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"8ff20d7c-298e-4f04-bf40-096a7758b19a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"fa7f664e-2706-4d40-81a8-d948c2af5d48","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"09655cda-95ce-4ddf-90ff-ad63afeb3a12"}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"09655cda-95ce-4ddf-90ff-ad63afeb3a12"}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"09655cda-95ce-4ddf-90ff-ad63afeb3a12"}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"abfed6bf-2b02-400b-95fd-e2b92c24e779","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"abfed6bf-2b02-400b-95fd-e2b92c24e779","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"42f2e0a4-a913-4ed7-9a05-19554961cc5a","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a1e387e3-60d0-416a-b6b2-91a43bf7017c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a1e387e3-60d0-416a-b6b2-91a43bf7017c","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a1e387e3-60d0-416a-b6b2-91a43bf7017c","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a1e387e3-60d0-416a-b6b2-91a43bf7017c"}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a1e387e3-60d0-416a-b6b2-91a43bf7017c","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:00.941Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a1e387e3-60d0-416a-b6b2-91a43bf7017c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:56:00.946Z","logger":"raycluster-resource","msg":"default","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.966Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-qvndg","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T09:56:00.966Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"09655cda-95ce-4ddf-90ff-ad63afeb3a12","name":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:00.966Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"09655cda-95ce-4ddf-90ff-ad63afeb3a12","error":"head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"error","ts":"2026-09-18T09:56:00.966Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"09655cda-95ce-4ddf-90ff-ad63afeb3a12","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:00.966Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"09655cda-95ce-4ddf-90ff-ad63afeb3a12","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"info","ts":"2026-09-18T09:56:00.966Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"09655cda-95ce-4ddf-90ff-ad63afeb3a12","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:00.966Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"09655cda-95ce-4ddf-90ff-ad63afeb3a12","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:00.966Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"09655cda-95ce-4ddf-90ff-ad63afeb3a12","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:00.966Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"09655cda-95ce-4ddf-90ff-ad63afeb3a12","name":"fail-qvndg","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:56:00.967Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"abfed6bf-2b02-400b-95fd-e2b92c24e779","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.967Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"abfed6bf-2b02-400b-95fd-e2b92c24e779","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.975Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"596a7b80-ebf1-47cd-9b92-121922a0f2c7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:00.975Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"596a7b80-ebf1-47cd-9b92-121922a0f2c7","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.975Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"596a7b80-ebf1-47cd-9b92-121922a0f2c7","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:00.975Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"596a7b80-ebf1-47cd-9b92-121922a0f2c7"}
{"level":"info","ts":"2026-09-18T09:56:00.975Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"596a7b80-ebf1-47cd-9b92-121922a0f2c7","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:00.975Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"596a7b80-ebf1-47cd-9b92-121922a0f2c7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:56:00.977Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"42f2e0a4-a913-4ed7-9a05-19554961cc5a","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.977Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"0d507f6e-613d-4c5e-9949-25baff2befc2","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.978Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"abfed6bf-2b02-400b-95fd-e2b92c24e779","rayCluster":"fail-qvndg","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T09:56:00.978Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"abfed6bf-2b02-400b-95fd-e2b92c24e779","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.979Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"09655cda-95ce-4ddf-90ff-ad63afeb3a12","name":"fail-qvndg","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-qvndg\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"fail-qvndg"}}
{"level":"info","ts":"2026-09-18T09:56:00.979Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"09655cda-95ce-4ddf-90ff-ad63afeb3a12"}
{"level":"error","ts":"2026-09-18T09:56:00.979Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"09655cda-95ce-4ddf-90ff-ad63afeb3a12","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:00.979Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0b31d523-6fc0-4d9b-8643-709b2c4bc228"}
{"level":"info","ts":"2026-09-18T09:56:00.979Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0b31d523-6fc0-4d9b-8643-709b2c4bc228"}
{"level":"info","ts":"2026-09-18T09:56:00.979Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0b31d523-6fc0-4d9b-8643-709b2c4bc228"}
{"level":"info","ts":"2026-09-18T09:56:00.979Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0b31d523-6fc0-4d9b-8643-709b2c4bc228","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:00.979Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0b31d523-6fc0-4d9b-8643-709b2c4bc228","error":"head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"error","ts":"2026-09-18T09:56:00.979Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0b31d523-6fc0-4d9b-8643-709b2c4bc228","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:00.979Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0b31d523-6fc0-4d9b-8643-709b2c4bc228","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"info","ts":"2026-09-18T09:56:00.979Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0b31d523-6fc0-4d9b-8643-709b2c4bc228","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:00.979Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0b31d523-6fc0-4d9b-8643-709b2c4bc228","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:00.979Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0b31d523-6fc0-4d9b-8643-709b2c4bc228","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:00.979Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0b31d523-6fc0-4d9b-8643-709b2c4bc228","name":"fail-qvndg","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:56:00.982Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"fa7f664e-2706-4d40-81a8-d948c2af5d48","name":"fail-qvndg-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:56:00.987Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"abfed6bf-2b02-400b-95fd-e2b92c24e779","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.989Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"0d507f6e-613d-4c5e-9949-25baff2befc2","name":"fail-qvndg-head"}
{"level":"info","ts":"2026-09-18T09:56:00.991Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0b31d523-6fc0-4d9b-8643-709b2c4bc228"}
{"level":"error","ts":"2026-09-18T09:56:00.991Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0b31d523-6fc0-4d9b-8643-709b2c4bc228","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:00.991Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"cfe155b9-6653-4b45-aea4-d688bef8704e"}
{"level":"info","ts":"2026-09-18T09:56:00.991Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"cfe155b9-6653-4b45-aea4-d688bef8704e"}
{"level":"info","ts":"2026-09-18T09:56:00.991Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"cfe155b9-6653-4b45-aea4-d688bef8704e"}
{"level":"info","ts":"2026-09-18T09:56:00.991Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"cfe155b9-6653-4b45-aea4-d688bef8704e","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:00.991Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"cfe155b9-6653-4b45-aea4-d688bef8704e","error":"head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"error","ts":"2026-09-18T09:56:00.991Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"cfe155b9-6653-4b45-aea4-d688bef8704e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:00.991Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"cfe155b9-6653-4b45-aea4-d688bef8704e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"info","ts":"2026-09-18T09:56:00.991Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"cfe155b9-6653-4b45-aea4-d688bef8704e","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:00.991Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"cfe155b9-6653-4b45-aea4-d688bef8704e","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:00.991Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"cfe155b9-6653-4b45-aea4-d688bef8704e","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:00.991Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"cfe155b9-6653-4b45-aea4-d688bef8704e","name":"fail-qvndg","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:56:00.991Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a427b2ef-c150-463a-9209-0c26d2925f21","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:00.991Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a427b2ef-c150-463a-9209-0c26d2925f21","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:00.991Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a427b2ef-c150-463a-9209-0c26d2925f21","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:01.066Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a427b2ef-c150-463a-9209-0c26d2925f21"}
{"level":"info","ts":"2026-09-18T09:56:01.066Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a427b2ef-c150-463a-9209-0c26d2925f21","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:01.066Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a427b2ef-c150-463a-9209-0c26d2925f21","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:01.066Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a427b2ef-c150-463a-9209-0c26d2925f21","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:56:01.066Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"fa7f664e-2706-4d40-81a8-d948c2af5d48","name":"kuberay-gateway-access","namespace":"test-ns-ltx56","operation":"created"}
{"level":"info","ts":"2026-09-18T09:56:01.066Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"abfed6bf-2b02-400b-95fd-e2b92c24e779","rayCluster":"fail-qvndg","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:56:01.071Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"0d507f6e-613d-4c5e-9949-25baff2befc2","name":"fail-qvndg-workers"}
{"level":"info","ts":"2026-09-18T09:56:01.071Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"0d507f6e-613d-4c5e-9949-25baff2befc2","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.071Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"60d39ce0-e81f-440a-bb34-a8c7a8bd550e","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.072Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"cfe155b9-6653-4b45-aea4-d688bef8704e","name":"fail-qvndg","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-qvndg\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"fail-qvndg"}}
{"level":"info","ts":"2026-09-18T09:56:01.072Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"cfe155b9-6653-4b45-aea4-d688bef8704e"}
{"level":"error","ts":"2026-09-18T09:56:01.072Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"cfe155b9-6653-4b45-aea4-d688bef8704e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:01.072Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"33750d57-9ca8-4472-b619-eb43c644bc3b"}
{"level":"info","ts":"2026-09-18T09:56:01.072Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"33750d57-9ca8-4472-b619-eb43c644bc3b"}
{"level":"info","ts":"2026-09-18T09:56:01.072Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"33750d57-9ca8-4472-b619-eb43c644bc3b"}
{"level":"info","ts":"2026-09-18T09:56:01.073Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"33750d57-9ca8-4472-b619-eb43c644bc3b","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:01.073Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"33750d57-9ca8-4472-b619-eb43c644bc3b","error":"head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"error","ts":"2026-09-18T09:56:01.073Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"33750d57-9ca8-4472-b619-eb43c644bc3b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:01.073Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"33750d57-9ca8-4472-b619-eb43c644bc3b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"info","ts":"2026-09-18T09:56:01.073Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"33750d57-9ca8-4472-b619-eb43c644bc3b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:01.073Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"33750d57-9ca8-4472-b619-eb43c644bc3b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:01.073Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"33750d57-9ca8-4472-b619-eb43c644bc3b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:01.073Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"33750d57-9ca8-4472-b619-eb43c644bc3b"}
{"level":"error","ts":"2026-09-18T09:56:01.073Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"33750d57-9ca8-4472-b619-eb43c644bc3b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:01.074Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"fa7f664e-2706-4d40-81a8-d948c2af5d48","name":"test-ns-ltx56-fail-qvndg","operation":"created"}
{"level":"info","ts":"2026-09-18T09:56:01.078Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"abfed6bf-2b02-400b-95fd-e2b92c24e779","rayCluster":"fail-qvndg","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:56:01.078Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"abfed6bf-2b02-400b-95fd-e2b92c24e779"}
{"level":"info","ts":"2026-09-18T09:56:01.078Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"291c9c20-15fd-407a-93ff-cff303d4d513","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.079Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"291c9c20-15fd-407a-93ff-cff303d4d513"}
{"level":"info","ts":"2026-09-18T09:56:01.083Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b9f72ada-ca68-4efa-80d3-34ed309fc249","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:01.083Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b9f72ada-ca68-4efa-80d3-34ed309fc249","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.083Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b9f72ada-ca68-4efa-80d3-34ed309fc249","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:01.083Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b9f72ada-ca68-4efa-80d3-34ed309fc249"}
{"level":"info","ts":"2026-09-18T09:56:01.083Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b9f72ada-ca68-4efa-80d3-34ed309fc249","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:01.083Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b9f72ada-ca68-4efa-80d3-34ed309fc249","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:01.084Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"fa7f664e-2706-4d40-81a8-d948c2af5d48","name":"kube-rbac-proxy-config-fail-qvndg","operation":"created"}
{"level":"info","ts":"2026-09-18T09:56:01.089Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"60d39ce0-e81f-440a-bb34-a8c7a8bd550e","name":"fail-qvndg-head"}
{"level":"info","ts":"2026-09-18T09:56:01.093Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a87e50e7-baaa-4b28-91a3-d0b0e4aebc01"}
{"level":"info","ts":"2026-09-18T09:56:01.093Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a87e50e7-baaa-4b28-91a3-d0b0e4aebc01"}
{"level":"info","ts":"2026-09-18T09:56:01.093Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a87e50e7-baaa-4b28-91a3-d0b0e4aebc01"}
{"level":"info","ts":"2026-09-18T09:56:01.093Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a87e50e7-baaa-4b28-91a3-d0b0e4aebc01","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:01.093Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a87e50e7-baaa-4b28-91a3-d0b0e4aebc01","error":"head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"error","ts":"2026-09-18T09:56:01.093Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a87e50e7-baaa-4b28-91a3-d0b0e4aebc01","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:01.093Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a87e50e7-baaa-4b28-91a3-d0b0e4aebc01","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"info","ts":"2026-09-18T09:56:01.093Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a87e50e7-baaa-4b28-91a3-d0b0e4aebc01","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:01.093Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a87e50e7-baaa-4b28-91a3-d0b0e4aebc01","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:01.093Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a87e50e7-baaa-4b28-91a3-d0b0e4aebc01","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:01.093Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a87e50e7-baaa-4b28-91a3-d0b0e4aebc01"}
{"level":"error","ts":"2026-09-18T09:56:01.093Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a87e50e7-baaa-4b28-91a3-d0b0e4aebc01","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:56:01.093Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"fa7f664e-2706-4d40-81a8-d948c2af5d48","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-qvndg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:01.093Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"fa7f664e-2706-4d40-81a8-d948c2af5d48"}
{"level":"error","ts":"2026-09-18T09:56:01.093Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"fa7f664e-2706-4d40-81a8-d948c2af5d48","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-qvndg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:01.093Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"03796aa3-6260-4ffa-97e1-205d7c8cf96a","namespacedName":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:01.093Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"03796aa3-6260-4ffa-97e1-205d7c8cf96a","mode":"IntegratedOAuth","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.093Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"03796aa3-6260-4ffa-97e1-205d7c8cf96a","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.172Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"60d39ce0-e81f-440a-bb34-a8c7a8bd550e","name":"fail-qvndg-workers"}
{"level":"info","ts":"2026-09-18T09:56:01.172Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"60d39ce0-e81f-440a-bb34-a8c7a8bd550e","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.173Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"92318897-9381-4172-aca6-b399ff182559"}
{"level":"info","ts":"2026-09-18T09:56:01.173Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"92318897-9381-4172-aca6-b399ff182559"}
{"level":"info","ts":"2026-09-18T09:56:01.173Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"92318897-9381-4172-aca6-b399ff182559"}
{"level":"info","ts":"2026-09-18T09:56:01.173Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"92318897-9381-4172-aca6-b399ff182559","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:01.173Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"92318897-9381-4172-aca6-b399ff182559","error":"head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"error","ts":"2026-09-18T09:56:01.173Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"92318897-9381-4172-aca6-b399ff182559","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:01.173Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"92318897-9381-4172-aca6-b399ff182559","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"info","ts":"2026-09-18T09:56:01.174Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"92318897-9381-4172-aca6-b399ff182559","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:01.174Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"92318897-9381-4172-aca6-b399ff182559","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:01.174Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"92318897-9381-4172-aca6-b399ff182559","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:01.174Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"92318897-9381-4172-aca6-b399ff182559"}
{"level":"error","ts":"2026-09-18T09:56:01.174Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"92318897-9381-4172-aca6-b399ff182559","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:01.176Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"03796aa3-6260-4ffa-97e1-205d7c8cf96a","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.177Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"80c6772d-4827-427d-96c4-4f1f9abc2279","namespacedName":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:01.177Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"80c6772d-4827-427d-96c4-4f1f9abc2279","mode":"IntegratedOAuth","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.177Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"80c6772d-4827-427d-96c4-4f1f9abc2279","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.177Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e51da6a9-ab52-4551-9ef4-99e989f8029a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:01.177Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e51da6a9-ab52-4551-9ef4-99e989f8029a","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.177Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e51da6a9-ab52-4551-9ef4-99e989f8029a","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:01.177Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"ab0d93d4-bda2-4173-9658-82364e746063","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.177Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e51da6a9-ab52-4551-9ef4-99e989f8029a"}
{"level":"info","ts":"2026-09-18T09:56:01.177Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e51da6a9-ab52-4551-9ef4-99e989f8029a","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:01.177Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e51da6a9-ab52-4551-9ef4-99e989f8029a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:01.177Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e51da6a9-ab52-4551-9ef4-99e989f8029a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:56:01.177Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"e2196222-f8fd-4857-8a7f-c3b0f9db2f30","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.177Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"e2196222-f8fd-4857-8a7f-c3b0f9db2f30"}
{"level":"error","ts":"2026-09-18T09:56:01.186Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"80c6772d-4827-427d-96c4-4f1f9abc2279","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-qvndg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:01.186Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"80c6772d-4827-427d-96c4-4f1f9abc2279"}
{"level":"error","ts":"2026-09-18T09:56:01.186Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"80c6772d-4827-427d-96c4-4f1f9abc2279","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-qvndg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:01.190Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dffdd89b-61c3-442a-a854-3875da7a680a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:01.190Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dffdd89b-61c3-442a-a854-3875da7a680a","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.190Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dffdd89b-61c3-442a-a854-3875da7a680a","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:01.190Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dffdd89b-61c3-442a-a854-3875da7a680a"}
{"level":"info","ts":"2026-09-18T09:56:01.190Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dffdd89b-61c3-442a-a854-3875da7a680a","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:01.190Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dffdd89b-61c3-442a-a854-3875da7a680a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:01.192Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"569d9d0f-25ee-4dea-96bd-638aa3ffb032","namespacedName":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:01.192Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"569d9d0f-25ee-4dea-96bd-638aa3ffb032","mode":"IntegratedOAuth","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.192Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"569d9d0f-25ee-4dea-96bd-638aa3ffb032","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.192Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"ab0d93d4-bda2-4173-9658-82364e746063","name":"fail-qvndg-head"}
{"level":"info","ts":"2026-09-18T09:56:01.202Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"569d9d0f-25ee-4dea-96bd-638aa3ffb032","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.205Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"ab0d93d4-bda2-4173-9658-82364e746063","name":"fail-qvndg-workers"}
{"level":"info","ts":"2026-09-18T09:56:01.205Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"ab0d93d4-bda2-4173-9658-82364e746063","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:01.334Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"b9d3b699-e60c-4ea5-83ef-f6f740f6afb4"}
{"level":"info","ts":"2026-09-18T09:56:01.334Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"b9d3b699-e60c-4ea5-83ef-f6f740f6afb4"}
{"level":"info","ts":"2026-09-18T09:56:01.334Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"b9d3b699-e60c-4ea5-83ef-f6f740f6afb4"}
{"level":"info","ts":"2026-09-18T09:56:01.334Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"b9d3b699-e60c-4ea5-83ef-f6f740f6afb4","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:01.334Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"b9d3b699-e60c-4ea5-83ef-f6f740f6afb4","error":"head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"error","ts":"2026-09-18T09:56:01.334Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"b9d3b699-e60c-4ea5-83ef-f6f740f6afb4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:01.334Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"b9d3b699-e60c-4ea5-83ef-f6f740f6afb4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"info","ts":"2026-09-18T09:56:01.334Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"b9d3b699-e60c-4ea5-83ef-f6f740f6afb4","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:01.334Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"b9d3b699-e60c-4ea5-83ef-f6f740f6afb4","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:01.334Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"b9d3b699-e60c-4ea5-83ef-f6f740f6afb4","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:01.334Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"b9d3b699-e60c-4ea5-83ef-f6f740f6afb4"}
{"level":"error","ts":"2026-09-18T09:56:01.334Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"b9d3b699-e60c-4ea5-83ef-f6f740f6afb4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:01.655Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9d992941-fdfa-456b-ac5c-b45930f38b9b"}
{"level":"info","ts":"2026-09-18T09:56:01.655Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9d992941-fdfa-456b-ac5c-b45930f38b9b"}
{"level":"info","ts":"2026-09-18T09:56:01.655Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9d992941-fdfa-456b-ac5c-b45930f38b9b"}
{"level":"info","ts":"2026-09-18T09:56:01.655Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9d992941-fdfa-456b-ac5c-b45930f38b9b","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:01.655Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9d992941-fdfa-456b-ac5c-b45930f38b9b","error":"head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"error","ts":"2026-09-18T09:56:01.655Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9d992941-fdfa-456b-ac5c-b45930f38b9b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:01.655Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9d992941-fdfa-456b-ac5c-b45930f38b9b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"info","ts":"2026-09-18T09:56:01.656Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9d992941-fdfa-456b-ac5c-b45930f38b9b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:01.656Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9d992941-fdfa-456b-ac5c-b45930f38b9b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:01.656Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9d992941-fdfa-456b-ac5c-b45930f38b9b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:01.656Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9d992941-fdfa-456b-ac5c-b45930f38b9b"}
{"level":"error","ts":"2026-09-18T09:56:01.656Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9d992941-fdfa-456b-ac5c-b45930f38b9b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:02.296Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9c8849b6-d71d-4b01-a861-f855f10af4d5"}
{"level":"info","ts":"2026-09-18T09:56:02.296Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9c8849b6-d71d-4b01-a861-f855f10af4d5"}
{"level":"info","ts":"2026-09-18T09:56:02.296Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9c8849b6-d71d-4b01-a861-f855f10af4d5"}
{"level":"info","ts":"2026-09-18T09:56:02.296Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9c8849b6-d71d-4b01-a861-f855f10af4d5","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:02.296Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9c8849b6-d71d-4b01-a861-f855f10af4d5","error":"head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"error","ts":"2026-09-18T09:56:02.296Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9c8849b6-d71d-4b01-a861-f855f10af4d5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:02.296Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9c8849b6-d71d-4b01-a861-f855f10af4d5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found"}
{"level":"info","ts":"2026-09-18T09:56:02.296Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9c8849b6-d71d-4b01-a861-f855f10af4d5","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:02.296Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9c8849b6-d71d-4b01-a861-f855f10af4d5","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:02.296Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9c8849b6-d71d-4b01-a861-f855f10af4d5","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-qvndg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:56:02.296Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9c8849b6-d71d-4b01-a861-f855f10af4d5"}
{"level":"error","ts":"2026-09-18T09:56:02.296Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9c8849b6-d71d-4b01-a861-f855f10af4d5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-qvndg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:03.577Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b"}
{"level":"info","ts":"2026-09-18T09:56:03.577Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b"}
{"level":"info","ts":"2026-09-18T09:56:03.577Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b"}
{"level":"info","ts":"2026-09-18T09:56:03.577Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:03.577Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b"}
{"level":"info","ts":"2026-09-18T09:56:03.577Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:56:03.577Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b"}
{"level":"info","ts":"2026-09-18T09:56:03.577Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","mode":"IntegratedOAuth","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:03.577Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","cluster":"fail-qvndg","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:56:03.577Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","cluster":"fail-qvndg","authType":"IntegratedOAuth","serviceAccount":"fail-qvndg-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:56:03.577Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-qvndg","ray.io/group":"headgroup","ray.io/identifier":"fail-qvndg-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:56:03.577Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:56:03.577Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:56:03.577Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:56:03.577Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","enabled":true}
{"level":"info","ts":"2026-09-18T09:56:03.597Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","name":"fail-qvndg-head-vlvq5"}
{"level":"info","ts":"2026-09-18T09:56:03.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:56:03.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:03.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:56:03.597Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:56:03.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:03.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:56:03.597Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b"}
{"level":"info","ts":"2026-09-18T09:56:03.597Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","nodeType":"worker","rayStartParams":{"address":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:56:03.597Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","rayStartCmd":"ray start  --address=fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:56:03.597Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:56:03.597Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","enabled":true}
{"level":"info","ts":"2026-09-18T09:56:03.616Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","name":"fail-qvndg-small-group-worker-dktj6"}
{"level":"info","ts":"2026-09-18T09:56:03.617Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:03.617Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:03.617Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"af79bb97-11e1-4e6e-acd6-cf9cab59c74b","name":"fail-qvndg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc"}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc"}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc"}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b0b72fe0-6c88-4e79-a182-a9b2ca8daaa5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b0b72fe0-6c88-4e79-a182-a9b2ca8daaa5","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b0b72fe0-6c88-4e79-a182-a9b2ca8daaa5","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc","name":"fail-qvndg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b0b72fe0-6c88-4e79-a182-a9b2ca8daaa5"}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b0b72fe0-6c88-4e79-a182-a9b2ca8daaa5","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:03.635Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b0b72fe0-6c88-4e79-a182-a9b2ca8daaa5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:03.636Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b0b72fe0-6c88-4e79-a182-a9b2ca8daaa5","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:56:03.636Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"baf0c5d6-9c00-41fa-b4f1-4ff1a8ee5201","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:03.636Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"2beb5cc5-6f28-438b-8bc0-ade808b10b47","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:03.636Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"baf0c5d6-9c00-41fa-b4f1-4ff1a8ee5201","rayCluster":"fail-qvndg","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:56:03.666Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc","name":"fail-qvndg","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-qvndg\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"fail-qvndg"}}
{"level":"info","ts":"2026-09-18T09:56:03.666Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc"}
{"level":"error","ts":"2026-09-18T09:56:03.666Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f5d85ea7-e4ce-4e48-b2bb-926b1bf799cc","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-qvndg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:56:03.666Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"6dd7c391-c8db-454d-8e18-fa5bc4663165"}
{"level":"info","ts":"2026-09-18T09:56:03.666Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"6dd7c391-c8db-454d-8e18-fa5bc4663165"}
{"level":"info","ts":"2026-09-18T09:56:03.666Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"6dd7c391-c8db-454d-8e18-fa5bc4663165"}
{"level":"info","ts":"2026-09-18T09:56:03.666Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"6dd7c391-c8db-454d-8e18-fa5bc4663165","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:03.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"6dd7c391-c8db-454d-8e18-fa5bc4663165","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:56:03.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"6dd7c391-c8db-454d-8e18-fa5bc4663165","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:03.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"6dd7c391-c8db-454d-8e18-fa5bc4663165","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:56:03.667Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"975433c6-90aa-4f20-b32b-ed55232a35b4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:03.667Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"975433c6-90aa-4f20-b32b-ed55232a35b4","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:03.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"6dd7c391-c8db-454d-8e18-fa5bc4663165","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:03.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"6dd7c391-c8db-454d-8e18-fa5bc4663165","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:03.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"6dd7c391-c8db-454d-8e18-fa5bc4663165","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:56:03.667Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"975433c6-90aa-4f20-b32b-ed55232a35b4","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:03.667Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"6dd7c391-c8db-454d-8e18-fa5bc4663165","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:56:03.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"6dd7c391-c8db-454d-8e18-fa5bc4663165","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:03.667Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"6dd7c391-c8db-454d-8e18-fa5bc4663165","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:03.667Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"975433c6-90aa-4f20-b32b-ed55232a35b4"}
{"level":"info","ts":"2026-09-18T09:56:03.667Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"975433c6-90aa-4f20-b32b-ed55232a35b4","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:03.667Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"6dd7c391-c8db-454d-8e18-fa5bc4663165","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:03.667Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"975433c6-90aa-4f20-b32b-ed55232a35b4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:03.667Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"6dd7c391-c8db-454d-8e18-fa5bc4663165","name":"fail-qvndg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:56:03.671Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"2beb5cc5-6f28-438b-8bc0-ade808b10b47","name":"fail-qvndg-head"}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"ff7276ab-d8c5-4bb5-946d-d58df0379144","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"ff7276ab-d8c5-4bb5-946d-d58df0379144","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"ff7276ab-d8c5-4bb5-946d-d58df0379144","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"d122c6b2-1837-4403-baf4-09ed79af3ccb","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"ff7276ab-d8c5-4bb5-946d-d58df0379144"}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"ff7276ab-d8c5-4bb5-946d-d58df0379144","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"ff7276ab-d8c5-4bb5-946d-d58df0379144","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"ff7276ab-d8c5-4bb5-946d-d58df0379144","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"d122c6b2-1837-4403-baf4-09ed79af3ccb","rayCluster":"fail-qvndg","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"2a907dda-c885-45d4-ac7e-098addef7d54"}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"2a907dda-c885-45d4-ac7e-098addef7d54"}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"2a907dda-c885-45d4-ac7e-098addef7d54"}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"2a907dda-c885-45d4-ac7e-098addef7d54","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"2a907dda-c885-45d4-ac7e-098addef7d54","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"2a907dda-c885-45d4-ac7e-098addef7d54","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"2a907dda-c885-45d4-ac7e-098addef7d54","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"2a907dda-c885-45d4-ac7e-098addef7d54","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"2a907dda-c885-45d4-ac7e-098addef7d54","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"2a907dda-c885-45d4-ac7e-098addef7d54","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"2a907dda-c885-45d4-ac7e-098addef7d54","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"2a907dda-c885-45d4-ac7e-098addef7d54","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"2a907dda-c885-45d4-ac7e-098addef7d54","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:03.681Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"2a907dda-c885-45d4-ac7e-098addef7d54","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:03.682Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"2a907dda-c885-45d4-ac7e-098addef7d54","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:56:03.682Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"2a907dda-c885-45d4-ac7e-098addef7d54","seconds":300}
{"level":"info","ts":"2026-09-18T09:56:03.685Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"2beb5cc5-6f28-438b-8bc0-ade808b10b47","name":"fail-qvndg-workers"}
{"level":"info","ts":"2026-09-18T09:56:03.685Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"2beb5cc5-6f28-438b-8bc0-ade808b10b47","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:03.685Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"5563ca1f-0bbb-4ce3-9955-8ad8c282497c","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:03.695Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"9311c8af-b17f-4056-80eb-85ffc98d6a66","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:03.695Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"9311c8af-b17f-4056-80eb-85ffc98d6a66","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:03.695Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"9311c8af-b17f-4056-80eb-85ffc98d6a66","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:03.695Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"9311c8af-b17f-4056-80eb-85ffc98d6a66"}
{"level":"info","ts":"2026-09-18T09:56:03.695Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"9311c8af-b17f-4056-80eb-85ffc98d6a66","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:03.695Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"9311c8af-b17f-4056-80eb-85ffc98d6a66","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:03.697Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"5563ca1f-0bbb-4ce3-9955-8ad8c282497c","name":"fail-qvndg-head"}
{"level":"info","ts":"2026-09-18T09:56:03.710Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"5563ca1f-0bbb-4ce3-9955-8ad8c282497c","name":"fail-qvndg-workers"}
{"level":"info","ts":"2026-09-18T09:56:03.710Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"5563ca1f-0bbb-4ce3-9955-8ad8c282497c","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:03.892Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter","namespace":"test-ns-ltx56"},"reconcileID":"94fac2d0-8eb7-4dcd-930a-f8198b78a797"}
{"level":"info","ts":"2026-09-18T09:56:03.917Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"883642a2-ca50-426e-ad13-c6a1ab7e57fe","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:03.917Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"883642a2-ca50-426e-ad13-c6a1ab7e57fe","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:03.917Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"883642a2-ca50-426e-ad13-c6a1ab7e57fe","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:03.917Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"883642a2-ca50-426e-ad13-c6a1ab7e57fe"}
{"level":"info","ts":"2026-09-18T09:56:03.917Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"883642a2-ca50-426e-ad13-c6a1ab7e57fe","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:03.917Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"883642a2-ca50-426e-ad13-c6a1ab7e57fe","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:04.200Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"73c3a292-1ff2-4333-a0d4-a82a3bee5784"}
{"level":"info","ts":"2026-09-18T09:56:04.200Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"73c3a292-1ff2-4333-a0d4-a82a3bee5784"}
{"level":"info","ts":"2026-09-18T09:56:04.200Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"73c3a292-1ff2-4333-a0d4-a82a3bee5784"}
{"level":"info","ts":"2026-09-18T09:56:04.200Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"73c3a292-1ff2-4333-a0d4-a82a3bee5784","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:04.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"73c3a292-1ff2-4333-a0d4-a82a3bee5784","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:56:04.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"73c3a292-1ff2-4333-a0d4-a82a3bee5784","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:04.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"73c3a292-1ff2-4333-a0d4-a82a3bee5784","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:56:04.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"73c3a292-1ff2-4333-a0d4-a82a3bee5784","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:04.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"73c3a292-1ff2-4333-a0d4-a82a3bee5784","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:04.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"73c3a292-1ff2-4333-a0d4-a82a3bee5784","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:56:04.200Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"73c3a292-1ff2-4333-a0d4-a82a3bee5784","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:56:04.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"73c3a292-1ff2-4333-a0d4-a82a3bee5784","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:04.200Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"73c3a292-1ff2-4333-a0d4-a82a3bee5784","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:04.200Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"73c3a292-1ff2-4333-a0d4-a82a3bee5784","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:04.200Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"73c3a292-1ff2-4333-a0d4-a82a3bee5784","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:56:04.200Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"73c3a292-1ff2-4333-a0d4-a82a3bee5784","seconds":300}
{"level":"info","ts":"2026-09-18T09:56:04.214Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"ae8bb5e9-5916-4788-8e34-6f6376de7e12"}
{"level":"info","ts":"2026-09-18T09:56:04.214Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"ae8bb5e9-5916-4788-8e34-6f6376de7e12"}
{"level":"info","ts":"2026-09-18T09:56:04.214Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"ae8bb5e9-5916-4788-8e34-6f6376de7e12"}
{"level":"info","ts":"2026-09-18T09:56:04.214Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"ae8bb5e9-5916-4788-8e34-6f6376de7e12","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:04.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"ae8bb5e9-5916-4788-8e34-6f6376de7e12","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:56:04.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"ae8bb5e9-5916-4788-8e34-6f6376de7e12","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:04.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"ae8bb5e9-5916-4788-8e34-6f6376de7e12","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:56:04.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"ae8bb5e9-5916-4788-8e34-6f6376de7e12","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:04.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"ae8bb5e9-5916-4788-8e34-6f6376de7e12","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:04.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"ae8bb5e9-5916-4788-8e34-6f6376de7e12","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:56:04.214Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"ae8bb5e9-5916-4788-8e34-6f6376de7e12","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:56:04.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"ae8bb5e9-5916-4788-8e34-6f6376de7e12","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:04.214Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"ae8bb5e9-5916-4788-8e34-6f6376de7e12","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:04.214Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"ae8bb5e9-5916-4788-8e34-6f6376de7e12","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:04.214Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"ae8bb5e9-5916-4788-8e34-6f6376de7e12","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:56:04.214Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"ae8bb5e9-5916-4788-8e34-6f6376de7e12","seconds":300}
{"level":"info","ts":"2026-09-18T09:56:04.793Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"394517a7-8168-4bd7-9408-296bbfb9babc"}
{"level":"info","ts":"2026-09-18T09:56:04.793Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"394517a7-8168-4bd7-9408-296bbfb9babc"}
{"level":"info","ts":"2026-09-18T09:56:04.793Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"394517a7-8168-4bd7-9408-296bbfb9babc"}
{"level":"info","ts":"2026-09-18T09:56:04.793Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"394517a7-8168-4bd7-9408-296bbfb9babc","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:04.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"394517a7-8168-4bd7-9408-296bbfb9babc","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:56:04.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"394517a7-8168-4bd7-9408-296bbfb9babc","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:04.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"394517a7-8168-4bd7-9408-296bbfb9babc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:56:04.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"394517a7-8168-4bd7-9408-296bbfb9babc","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:04.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"394517a7-8168-4bd7-9408-296bbfb9babc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:04.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"394517a7-8168-4bd7-9408-296bbfb9babc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:56:04.794Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"394517a7-8168-4bd7-9408-296bbfb9babc","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:56:04.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"394517a7-8168-4bd7-9408-296bbfb9babc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:04.794Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"394517a7-8168-4bd7-9408-296bbfb9babc","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:04.794Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"394517a7-8168-4bd7-9408-296bbfb9babc","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:04.794Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"394517a7-8168-4bd7-9408-296bbfb9babc","name":"fail-qvndg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:56:04.808Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"88e3f103-0a03-4070-9101-cf4798134b1e","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:04.808Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"42e4911d-6a0a-4f57-b71a-6e768e8badd7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:04.808Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"42e4911d-6a0a-4f57-b71a-6e768e8badd7","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:04.808Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"42e4911d-6a0a-4f57-b71a-6e768e8badd7","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:04.808Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"88e3f103-0a03-4070-9101-cf4798134b1e","pod":"fail-qvndg-head-vlvq5","ip":"10.133.0.39"}
{"level":"info","ts":"2026-09-18T09:56:04.808Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"42e4911d-6a0a-4f57-b71a-6e768e8badd7"}
{"level":"info","ts":"2026-09-18T09:56:04.808Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"42e4911d-6a0a-4f57-b71a-6e768e8badd7","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:04.808Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"42e4911d-6a0a-4f57-b71a-6e768e8badd7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:04.808Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"42e4911d-6a0a-4f57-b71a-6e768e8badd7","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:56:04.808Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"fad8caa1-0f85-4516-a69c-18d76c098d52","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:04.820Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"fad8caa1-0f85-4516-a69c-18d76c098d52","name":"fail-qvndg-head"}
{"level":"info","ts":"2026-09-18T09:56:04.821Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"88e3f103-0a03-4070-9101-cf4798134b1e","certificate":"ray-head-cert-fail-qvndg","dnsNames":["fail-qvndg-head-svc","localhost","fail-qvndg-head-svc.test-ns-ltx56.svc","fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.39","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:56:04.822Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"c778a661-6871-47d9-acd4-473e220395b3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:04.822Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"c778a661-6871-47d9-acd4-473e220395b3","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:04.822Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"c778a661-6871-47d9-acd4-473e220395b3","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:04.823Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"c778a661-6871-47d9-acd4-473e220395b3"}
{"level":"info","ts":"2026-09-18T09:56:04.823Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"c778a661-6871-47d9-acd4-473e220395b3","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:04.823Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"c778a661-6871-47d9-acd4-473e220395b3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:04.832Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"fad8caa1-0f85-4516-a69c-18d76c098d52","name":"fail-qvndg-workers"}
{"level":"info","ts":"2026-09-18T09:56:04.832Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"fad8caa1-0f85-4516-a69c-18d76c098d52","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:04.832Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"88e3f103-0a03-4070-9101-cf4798134b1e","certificate":"ray-worker-cert-fail-qvndg","dnsNames":["fail-qvndg-worker-svc","localhost","fail-qvndg-small-group","fail-qvndg-worker-svc.test-ns-ltx56.svc","fail-qvndg-worker-svc.test-ns-ltx56.svc.cluster.local","*.fail-qvndg-worker-svc.test-ns-ltx56.svc","*.fail-qvndg-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.39","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:56:04.833Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"88e3f103-0a03-4070-9101-cf4798134b1e","rayCluster":"fail-qvndg","podIPs":["10.133.0.39"]}
{"level":"info","ts":"2026-09-18T09:56:05.142Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"63b80d52-d039-4cf2-98de-bff16ebf9d5d"}
{"level":"info","ts":"2026-09-18T09:56:05.142Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"63b80d52-d039-4cf2-98de-bff16ebf9d5d"}
{"level":"info","ts":"2026-09-18T09:56:05.142Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"63b80d52-d039-4cf2-98de-bff16ebf9d5d"}
{"level":"info","ts":"2026-09-18T09:56:05.142Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"63b80d52-d039-4cf2-98de-bff16ebf9d5d","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:05.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"63b80d52-d039-4cf2-98de-bff16ebf9d5d","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:56:05.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"63b80d52-d039-4cf2-98de-bff16ebf9d5d","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:05.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"63b80d52-d039-4cf2-98de-bff16ebf9d5d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:56:05.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"63b80d52-d039-4cf2-98de-bff16ebf9d5d","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:05.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"63b80d52-d039-4cf2-98de-bff16ebf9d5d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:05.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"63b80d52-d039-4cf2-98de-bff16ebf9d5d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:56:05.143Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"63b80d52-d039-4cf2-98de-bff16ebf9d5d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:56:05.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"63b80d52-d039-4cf2-98de-bff16ebf9d5d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:05.143Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"63b80d52-d039-4cf2-98de-bff16ebf9d5d","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:05.143Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"63b80d52-d039-4cf2-98de-bff16ebf9d5d","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:05.143Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"63b80d52-d039-4cf2-98de-bff16ebf9d5d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:56:05.143Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"63b80d52-d039-4cf2-98de-bff16ebf9d5d","seconds":300}
{"level":"info","ts":"2026-09-18T09:56:05.682Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a9478f71-9ccf-49db-a2f6-735e5dce1635"}
{"level":"info","ts":"2026-09-18T09:56:05.682Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a9478f71-9ccf-49db-a2f6-735e5dce1635"}
{"level":"info","ts":"2026-09-18T09:56:05.682Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a9478f71-9ccf-49db-a2f6-735e5dce1635"}
{"level":"info","ts":"2026-09-18T09:56:05.682Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a9478f71-9ccf-49db-a2f6-735e5dce1635","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:05.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a9478f71-9ccf-49db-a2f6-735e5dce1635","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:56:05.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a9478f71-9ccf-49db-a2f6-735e5dce1635","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:05.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a9478f71-9ccf-49db-a2f6-735e5dce1635","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:56:05.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a9478f71-9ccf-49db-a2f6-735e5dce1635","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:05.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a9478f71-9ccf-49db-a2f6-735e5dce1635","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:05.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a9478f71-9ccf-49db-a2f6-735e5dce1635","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:56:05.682Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a9478f71-9ccf-49db-a2f6-735e5dce1635","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:56:05.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a9478f71-9ccf-49db-a2f6-735e5dce1635","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:05.682Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a9478f71-9ccf-49db-a2f6-735e5dce1635","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:05.682Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a9478f71-9ccf-49db-a2f6-735e5dce1635","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:05.682Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a9478f71-9ccf-49db-a2f6-735e5dce1635","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:56:05.682Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"a9478f71-9ccf-49db-a2f6-735e5dce1635","seconds":300}
{"level":"info","ts":"2026-09-18T09:56:06.918Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3ce74293-3e05-4ed2-a5f2-900740c836ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:06.918Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3ce74293-3e05-4ed2-a5f2-900740c836ff","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:06.918Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3ce74293-3e05-4ed2-a5f2-900740c836ff","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:06.918Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3ce74293-3e05-4ed2-a5f2-900740c836ff"}
{"level":"info","ts":"2026-09-18T09:56:06.918Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3ce74293-3e05-4ed2-a5f2-900740c836ff","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:06.918Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3ce74293-3e05-4ed2-a5f2-900740c836ff","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:09.817Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f2b88bfb-0613-407b-bd57-a82c3f783251"}
{"level":"info","ts":"2026-09-18T09:56:09.817Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f2b88bfb-0613-407b-bd57-a82c3f783251"}
{"level":"info","ts":"2026-09-18T09:56:09.817Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f2b88bfb-0613-407b-bd57-a82c3f783251"}
{"level":"info","ts":"2026-09-18T09:56:09.817Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f2b88bfb-0613-407b-bd57-a82c3f783251","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:09.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f2b88bfb-0613-407b-bd57-a82c3f783251","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:56:09.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f2b88bfb-0613-407b-bd57-a82c3f783251","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:09.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f2b88bfb-0613-407b-bd57-a82c3f783251","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:56:09.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f2b88bfb-0613-407b-bd57-a82c3f783251","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:09.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f2b88bfb-0613-407b-bd57-a82c3f783251","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:09.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f2b88bfb-0613-407b-bd57-a82c3f783251","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:56:09.818Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f2b88bfb-0613-407b-bd57-a82c3f783251","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:56:09.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f2b88bfb-0613-407b-bd57-a82c3f783251","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:09.818Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f2b88bfb-0613-407b-bd57-a82c3f783251","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:09.818Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f2b88bfb-0613-407b-bd57-a82c3f783251","podName":"fail-qvndg-head-vlvq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:09.818Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f2b88bfb-0613-407b-bd57-a82c3f783251","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:56:09.818Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f2b88bfb-0613-407b-bd57-a82c3f783251","seconds":300}
{"level":"info","ts":"2026-09-18T09:56:09.919Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a3114bdb-9b17-4d83-8bd0-97faa7eec99e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:09.919Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a3114bdb-9b17-4d83-8bd0-97faa7eec99e","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:09.920Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a3114bdb-9b17-4d83-8bd0-97faa7eec99e","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:09.920Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a3114bdb-9b17-4d83-8bd0-97faa7eec99e"}
{"level":"info","ts":"2026-09-18T09:56:09.920Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a3114bdb-9b17-4d83-8bd0-97faa7eec99e","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:09.920Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a3114bdb-9b17-4d83-8bd0-97faa7eec99e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:10.829Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"452061bb-35cd-44c0-93a4-619d1b0899df"}
{"level":"info","ts":"2026-09-18T09:56:10.829Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"452061bb-35cd-44c0-93a4-619d1b0899df"}
{"level":"info","ts":"2026-09-18T09:56:10.829Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"452061bb-35cd-44c0-93a4-619d1b0899df"}
{"level":"info","ts":"2026-09-18T09:56:10.829Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"452061bb-35cd-44c0-93a4-619d1b0899df","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:10.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"452061bb-35cd-44c0-93a4-619d1b0899df","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:56:10.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"452061bb-35cd-44c0-93a4-619d1b0899df","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:10.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"452061bb-35cd-44c0-93a4-619d1b0899df","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:56:10.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"452061bb-35cd-44c0-93a4-619d1b0899df","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:10.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"452061bb-35cd-44c0-93a4-619d1b0899df","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:10.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"452061bb-35cd-44c0-93a4-619d1b0899df","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:56:10.830Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"452061bb-35cd-44c0-93a4-619d1b0899df","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:56:10.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"452061bb-35cd-44c0-93a4-619d1b0899df","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:10.830Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"452061bb-35cd-44c0-93a4-619d1b0899df","podName":"fail-qvndg-head-vlvq5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:56:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:56:10.830Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"452061bb-35cd-44c0-93a4-619d1b0899df","podName":"fail-qvndg-head-vlvq5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:56:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:56:10.830Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"452061bb-35cd-44c0-93a4-619d1b0899df","name":"fail-qvndg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:56:10.845Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"e890fcc8-b190-4d27-97ae-d7f896eb836f","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:10.845Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"89a89445-ba63-4891-b8c9-64cc4060583b","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:10.845Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"e890fcc8-b190-4d27-97ae-d7f896eb836f","pod":"fail-qvndg-small-group-worker-dktj6","ip":"10.132.0.39"}
{"level":"info","ts":"2026-09-18T09:56:10.845Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"e890fcc8-b190-4d27-97ae-d7f896eb836f","pod":"fail-qvndg-head-vlvq5","ip":"10.133.0.39"}
{"level":"info","ts":"2026-09-18T09:56:10.845Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"cb395ca5-6bde-4a8b-bbf8-0adf47fc88fa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:10.845Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"cb395ca5-6bde-4a8b-bbf8-0adf47fc88fa","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:10.845Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"cb395ca5-6bde-4a8b-bbf8-0adf47fc88fa","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:10.845Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"cb395ca5-6bde-4a8b-bbf8-0adf47fc88fa"}
{"level":"info","ts":"2026-09-18T09:56:10.845Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"cb395ca5-6bde-4a8b-bbf8-0adf47fc88fa","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:10.845Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"cb395ca5-6bde-4a8b-bbf8-0adf47fc88fa","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:10.845Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"cb395ca5-6bde-4a8b-bbf8-0adf47fc88fa","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:56:10.857Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"e890fcc8-b190-4d27-97ae-d7f896eb836f","certificate":"ray-head-cert-fail-qvndg","dnsNames":["fail-qvndg-head-svc","localhost","fail-qvndg-head-svc.test-ns-ltx56.svc","fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.39","10.133.0.39","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:56:10.861Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"89a89445-ba63-4891-b8c9-64cc4060583b","name":"fail-qvndg-head"}
{"level":"info","ts":"2026-09-18T09:56:10.863Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dd9a3688-a9dd-42c1-ab66-027d8d93963a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:10.863Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dd9a3688-a9dd-42c1-ab66-027d8d93963a","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:10.863Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dd9a3688-a9dd-42c1-ab66-027d8d93963a","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:10.863Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dd9a3688-a9dd-42c1-ab66-027d8d93963a"}
{"level":"info","ts":"2026-09-18T09:56:10.863Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dd9a3688-a9dd-42c1-ab66-027d8d93963a","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:10.863Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dd9a3688-a9dd-42c1-ab66-027d8d93963a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:10.873Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"e890fcc8-b190-4d27-97ae-d7f896eb836f","certificate":"ray-worker-cert-fail-qvndg","dnsNames":["fail-qvndg-worker-svc","localhost","fail-qvndg-small-group","fail-qvndg-worker-svc.test-ns-ltx56.svc","fail-qvndg-worker-svc.test-ns-ltx56.svc.cluster.local","*.fail-qvndg-worker-svc.test-ns-ltx56.svc","*.fail-qvndg-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.39","10.133.0.39","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:56:10.873Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"e890fcc8-b190-4d27-97ae-d7f896eb836f","rayCluster":"fail-qvndg","podIPs":["10.132.0.39","10.133.0.39"]}
{"level":"info","ts":"2026-09-18T09:56:10.874Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"89a89445-ba63-4891-b8c9-64cc4060583b","name":"fail-qvndg-workers"}
{"level":"info","ts":"2026-09-18T09:56:10.874Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"89a89445-ba63-4891-b8c9-64cc4060583b","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:12.845Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0f622bc7-d5b7-4830-9949-0bed2d3f947b"}
{"level":"info","ts":"2026-09-18T09:56:12.845Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0f622bc7-d5b7-4830-9949-0bed2d3f947b"}
{"level":"info","ts":"2026-09-18T09:56:12.845Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0f622bc7-d5b7-4830-9949-0bed2d3f947b"}
{"level":"info","ts":"2026-09-18T09:56:12.845Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0f622bc7-d5b7-4830-9949-0bed2d3f947b","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:12.845Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0f622bc7-d5b7-4830-9949-0bed2d3f947b","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:56:12.845Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0f622bc7-d5b7-4830-9949-0bed2d3f947b","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:12.845Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0f622bc7-d5b7-4830-9949-0bed2d3f947b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:56:12.845Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0f622bc7-d5b7-4830-9949-0bed2d3f947b","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:12.845Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0f622bc7-d5b7-4830-9949-0bed2d3f947b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:12.845Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0f622bc7-d5b7-4830-9949-0bed2d3f947b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:56:12.845Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0f622bc7-d5b7-4830-9949-0bed2d3f947b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:56:12.845Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0f622bc7-d5b7-4830-9949-0bed2d3f947b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:12.846Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0f622bc7-d5b7-4830-9949-0bed2d3f947b","podName":"fail-qvndg-head-vlvq5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:56:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:56:12.846Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0f622bc7-d5b7-4830-9949-0bed2d3f947b","podName":"fail-qvndg-head-vlvq5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:56:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:56:12.846Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0f622bc7-d5b7-4830-9949-0bed2d3f947b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:56:12.846Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0f622bc7-d5b7-4830-9949-0bed2d3f947b","seconds":300}
{"level":"info","ts":"2026-09-18T09:56:12.920Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"82485b38-2b1a-43b3-a386-6d287a43d1d0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:12.920Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"82485b38-2b1a-43b3-a386-6d287a43d1d0","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:12.920Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"82485b38-2b1a-43b3-a386-6d287a43d1d0","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:12.921Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"82485b38-2b1a-43b3-a386-6d287a43d1d0"}
{"level":"info","ts":"2026-09-18T09:56:12.921Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"82485b38-2b1a-43b3-a386-6d287a43d1d0","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:12.921Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"82485b38-2b1a-43b3-a386-6d287a43d1d0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:15.922Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dfe0a0ba-e6ad-4003-8dfd-e5b9af7522db","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:15.922Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dfe0a0ba-e6ad-4003-8dfd-e5b9af7522db","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:15.922Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dfe0a0ba-e6ad-4003-8dfd-e5b9af7522db","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:15.922Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dfe0a0ba-e6ad-4003-8dfd-e5b9af7522db"}
{"level":"info","ts":"2026-09-18T09:56:15.922Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dfe0a0ba-e6ad-4003-8dfd-e5b9af7522db","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:15.922Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dfe0a0ba-e6ad-4003-8dfd-e5b9af7522db","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:18.922Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d8d2fc00-e1c9-49f6-803c-0dc881fd799b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:18.922Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d8d2fc00-e1c9-49f6-803c-0dc881fd799b","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:18.922Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d8d2fc00-e1c9-49f6-803c-0dc881fd799b","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:18.923Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d8d2fc00-e1c9-49f6-803c-0dc881fd799b"}
{"level":"info","ts":"2026-09-18T09:56:18.923Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d8d2fc00-e1c9-49f6-803c-0dc881fd799b","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:18.923Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d8d2fc00-e1c9-49f6-803c-0dc881fd799b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:21.854Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9f05db5b-d22c-4d18-9189-e1ae9dc9335f"}
{"level":"info","ts":"2026-09-18T09:56:21.854Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9f05db5b-d22c-4d18-9189-e1ae9dc9335f"}
{"level":"info","ts":"2026-09-18T09:56:21.854Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9f05db5b-d22c-4d18-9189-e1ae9dc9335f"}
{"level":"info","ts":"2026-09-18T09:56:21.854Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9f05db5b-d22c-4d18-9189-e1ae9dc9335f","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:21.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9f05db5b-d22c-4d18-9189-e1ae9dc9335f","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:56:21.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9f05db5b-d22c-4d18-9189-e1ae9dc9335f","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:21.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9f05db5b-d22c-4d18-9189-e1ae9dc9335f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:56:21.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9f05db5b-d22c-4d18-9189-e1ae9dc9335f","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:21.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9f05db5b-d22c-4d18-9189-e1ae9dc9335f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:21.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9f05db5b-d22c-4d18-9189-e1ae9dc9335f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:56:21.855Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9f05db5b-d22c-4d18-9189-e1ae9dc9335f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:56:21.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9f05db5b-d22c-4d18-9189-e1ae9dc9335f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:21.855Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9f05db5b-d22c-4d18-9189-e1ae9dc9335f","podName":"fail-qvndg-small-group-worker-dktj6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:21.855Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9f05db5b-d22c-4d18-9189-e1ae9dc9335f","podName":"fail-qvndg-small-group-worker-dktj6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:21.855Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"9f05db5b-d22c-4d18-9189-e1ae9dc9335f","name":"fail-qvndg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:56:21.867Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"ac115d36-20fb-4b4a-ae06-23e54a16b72d","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:21.867Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"740f4060-2d3a-4761-9233-5ebdbfb2b82e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:21.867Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"740f4060-2d3a-4761-9233-5ebdbfb2b82e","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:21.867Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"740f4060-2d3a-4761-9233-5ebdbfb2b82e","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:21.868Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"ac115d36-20fb-4b4a-ae06-23e54a16b72d","pod":"fail-qvndg-head-vlvq5","ip":"10.133.0.39"}
{"level":"info","ts":"2026-09-18T09:56:21.868Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"ac115d36-20fb-4b4a-ae06-23e54a16b72d","pod":"fail-qvndg-small-group-worker-dktj6","ip":"10.132.0.39"}
{"level":"info","ts":"2026-09-18T09:56:21.868Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"ac115d36-20fb-4b4a-ae06-23e54a16b72d","rayCluster":"fail-qvndg","podIPs":["10.133.0.39","10.132.0.39"]}
{"level":"info","ts":"2026-09-18T09:56:21.868Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"740f4060-2d3a-4761-9233-5ebdbfb2b82e"}
{"level":"info","ts":"2026-09-18T09:56:21.868Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"740f4060-2d3a-4761-9233-5ebdbfb2b82e","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:21.868Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"740f4060-2d3a-4761-9233-5ebdbfb2b82e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:21.868Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"740f4060-2d3a-4761-9233-5ebdbfb2b82e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:56:21.868Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"0756c3c6-d934-4d91-a47f-d4d7cb3493d6","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:21.882Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"83356ffe-9023-44ec-a8ef-c5eb77b87b81","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:21.882Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"83356ffe-9023-44ec-a8ef-c5eb77b87b81","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:21.882Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"83356ffe-9023-44ec-a8ef-c5eb77b87b81","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:21.882Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"83356ffe-9023-44ec-a8ef-c5eb77b87b81"}
{"level":"info","ts":"2026-09-18T09:56:21.882Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"83356ffe-9023-44ec-a8ef-c5eb77b87b81","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:21.882Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"83356ffe-9023-44ec-a8ef-c5eb77b87b81","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:21.885Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"0756c3c6-d934-4d91-a47f-d4d7cb3493d6","name":"fail-qvndg-head"}
{"level":"info","ts":"2026-09-18T09:56:21.897Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"0756c3c6-d934-4d91-a47f-d4d7cb3493d6","name":"fail-qvndg-workers"}
{"level":"info","ts":"2026-09-18T09:56:21.897Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"0756c3c6-d934-4d91-a47f-d4d7cb3493d6","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:21.924Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"652ced2a-8b92-44c9-b4c7-f1e56ffaceb3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:21.924Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"652ced2a-8b92-44c9-b4c7-f1e56ffaceb3","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:21.924Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"652ced2a-8b92-44c9-b4c7-f1e56ffaceb3","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:21.924Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"652ced2a-8b92-44c9-b4c7-f1e56ffaceb3"}
{"level":"info","ts":"2026-09-18T09:56:21.924Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"652ced2a-8b92-44c9-b4c7-f1e56ffaceb3","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:21.924Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"652ced2a-8b92-44c9-b4c7-f1e56ffaceb3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:23.868Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"d9b0ef25-3f2a-4788-9ac5-bea995a7948b"}
{"level":"info","ts":"2026-09-18T09:56:23.868Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"d9b0ef25-3f2a-4788-9ac5-bea995a7948b"}
{"level":"info","ts":"2026-09-18T09:56:23.868Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"d9b0ef25-3f2a-4788-9ac5-bea995a7948b"}
{"level":"info","ts":"2026-09-18T09:56:23.868Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"d9b0ef25-3f2a-4788-9ac5-bea995a7948b","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:56:23.868Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"d9b0ef25-3f2a-4788-9ac5-bea995a7948b","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:56:23.869Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"d9b0ef25-3f2a-4788-9ac5-bea995a7948b","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:23.869Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"d9b0ef25-3f2a-4788-9ac5-bea995a7948b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:56:23.869Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"d9b0ef25-3f2a-4788-9ac5-bea995a7948b","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:56:23.869Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"d9b0ef25-3f2a-4788-9ac5-bea995a7948b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:23.869Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"d9b0ef25-3f2a-4788-9ac5-bea995a7948b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:56:23.869Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"d9b0ef25-3f2a-4788-9ac5-bea995a7948b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:56:23.869Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"d9b0ef25-3f2a-4788-9ac5-bea995a7948b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:56:23.869Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"d9b0ef25-3f2a-4788-9ac5-bea995a7948b","podName":"fail-qvndg-small-group-worker-dktj6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:23.869Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"d9b0ef25-3f2a-4788-9ac5-bea995a7948b","podName":"fail-qvndg-small-group-worker-dktj6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:56:23.869Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"d9b0ef25-3f2a-4788-9ac5-bea995a7948b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:56:23.869Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"d9b0ef25-3f2a-4788-9ac5-bea995a7948b","seconds":300}
{"level":"info","ts":"2026-09-18T09:56:24.925Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b55328f6-38e6-4c31-ad0b-33b003de5ebd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:24.925Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b55328f6-38e6-4c31-ad0b-33b003de5ebd","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:24.925Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b55328f6-38e6-4c31-ad0b-33b003de5ebd","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:24.925Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b55328f6-38e6-4c31-ad0b-33b003de5ebd"}
{"level":"info","ts":"2026-09-18T09:56:24.925Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b55328f6-38e6-4c31-ad0b-33b003de5ebd","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:24.925Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b55328f6-38e6-4c31-ad0b-33b003de5ebd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:27.926Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b7515306-2497-4e03-b1b6-ea9d5896ed41","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:27.926Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b7515306-2497-4e03-b1b6-ea9d5896ed41","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:27.926Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b7515306-2497-4e03-b1b6-ea9d5896ed41","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:27.926Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b7515306-2497-4e03-b1b6-ea9d5896ed41"}
{"level":"info","ts":"2026-09-18T09:56:27.926Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b7515306-2497-4e03-b1b6-ea9d5896ed41","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:27.926Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b7515306-2497-4e03-b1b6-ea9d5896ed41","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:30.927Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"256a7cd6-cdf4-438d-9424-7b5ebddb8f78","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:30.927Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"256a7cd6-cdf4-438d-9424-7b5ebddb8f78","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:30.927Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"256a7cd6-cdf4-438d-9424-7b5ebddb8f78","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:30.927Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"256a7cd6-cdf4-438d-9424-7b5ebddb8f78"}
{"level":"info","ts":"2026-09-18T09:56:30.927Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"256a7cd6-cdf4-438d-9424-7b5ebddb8f78","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:30.927Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"256a7cd6-cdf4-438d-9424-7b5ebddb8f78","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:31.079Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"6a15d6cf-fa36-40ee-bba3-0fe6c2bbbbff","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:31.079Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"6a15d6cf-fa36-40ee-bba3-0fe6c2bbbbff","pod":"fail-qvndg-head-vlvq5","ip":"10.133.0.39"}
{"level":"info","ts":"2026-09-18T09:56:31.079Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"6a15d6cf-fa36-40ee-bba3-0fe6c2bbbbff","pod":"fail-qvndg-small-group-worker-dktj6","ip":"10.132.0.39"}
{"level":"info","ts":"2026-09-18T09:56:31.079Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"6a15d6cf-fa36-40ee-bba3-0fe6c2bbbbff","rayCluster":"fail-qvndg","podIPs":["10.133.0.39","10.132.0.39"]}
{"level":"info","ts":"2026-09-18T09:56:33.927Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"f51d0bc4-8328-417a-a744-0c233ddb12fc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:33.927Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"f51d0bc4-8328-417a-a744-0c233ddb12fc","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:33.927Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"f51d0bc4-8328-417a-a744-0c233ddb12fc","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:33.928Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"f51d0bc4-8328-417a-a744-0c233ddb12fc"}
{"level":"info","ts":"2026-09-18T09:56:33.928Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"f51d0bc4-8328-417a-a744-0c233ddb12fc","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:33.928Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"f51d0bc4-8328-417a-a744-0c233ddb12fc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:36.929Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"7bfa9d53-cd73-4144-a9fe-e36340e940ae","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:36.929Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"7bfa9d53-cd73-4144-a9fe-e36340e940ae","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:36.929Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"7bfa9d53-cd73-4144-a9fe-e36340e940ae","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:36.929Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"7bfa9d53-cd73-4144-a9fe-e36340e940ae"}
{"level":"info","ts":"2026-09-18T09:56:36.929Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"7bfa9d53-cd73-4144-a9fe-e36340e940ae","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:36.929Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"7bfa9d53-cd73-4144-a9fe-e36340e940ae","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:39.929Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"74f1b497-b13f-4c7e-a17a-64f82e522cca","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:39.929Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"74f1b497-b13f-4c7e-a17a-64f82e522cca","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:39.929Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"74f1b497-b13f-4c7e-a17a-64f82e522cca","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:39.930Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"74f1b497-b13f-4c7e-a17a-64f82e522cca"}
{"level":"info","ts":"2026-09-18T09:56:39.930Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"74f1b497-b13f-4c7e-a17a-64f82e522cca","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:39.930Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"74f1b497-b13f-4c7e-a17a-64f82e522cca","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:42.931Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"fe1cbc3c-a818-470d-b9dd-e9836603379a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:42.931Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"fe1cbc3c-a818-470d-b9dd-e9836603379a","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:42.931Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"fe1cbc3c-a818-470d-b9dd-e9836603379a","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:42.931Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"fe1cbc3c-a818-470d-b9dd-e9836603379a"}
{"level":"info","ts":"2026-09-18T09:56:42.931Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"fe1cbc3c-a818-470d-b9dd-e9836603379a","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:42.931Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"fe1cbc3c-a818-470d-b9dd-e9836603379a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:45.932Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"ba56e3f7-4c7f-4178-9a30-198ecb98cfd4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:45.932Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"ba56e3f7-4c7f-4178-9a30-198ecb98cfd4","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:45.932Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"ba56e3f7-4c7f-4178-9a30-198ecb98cfd4","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:45.933Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"ba56e3f7-4c7f-4178-9a30-198ecb98cfd4"}
{"level":"info","ts":"2026-09-18T09:56:45.933Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"ba56e3f7-4c7f-4178-9a30-198ecb98cfd4","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:45.933Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"ba56e3f7-4c7f-4178-9a30-198ecb98cfd4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:48.933Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"09fe5eb3-f854-4d2c-99ab-0ff0dc1e7b9a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:48.933Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"09fe5eb3-f854-4d2c-99ab-0ff0dc1e7b9a","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:48.933Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"09fe5eb3-f854-4d2c-99ab-0ff0dc1e7b9a","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:48.933Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"09fe5eb3-f854-4d2c-99ab-0ff0dc1e7b9a"}
{"level":"info","ts":"2026-09-18T09:56:48.933Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"09fe5eb3-f854-4d2c-99ab-0ff0dc1e7b9a","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:48.933Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"09fe5eb3-f854-4d2c-99ab-0ff0dc1e7b9a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:51.934Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a9a00bcd-9231-4402-87b4-db52480a6233","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:51.934Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a9a00bcd-9231-4402-87b4-db52480a6233","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:51.934Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a9a00bcd-9231-4402-87b4-db52480a6233","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:51.934Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a9a00bcd-9231-4402-87b4-db52480a6233"}
{"level":"info","ts":"2026-09-18T09:56:51.934Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a9a00bcd-9231-4402-87b4-db52480a6233","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:51.934Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a9a00bcd-9231-4402-87b4-db52480a6233","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:54.935Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"61115c32-3797-4375-ae61-0ddbebc55c5d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:54.935Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"61115c32-3797-4375-ae61-0ddbebc55c5d","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:54.935Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"61115c32-3797-4375-ae61-0ddbebc55c5d","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:54.935Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"61115c32-3797-4375-ae61-0ddbebc55c5d"}
{"level":"info","ts":"2026-09-18T09:56:54.935Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"61115c32-3797-4375-ae61-0ddbebc55c5d","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:54.935Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"61115c32-3797-4375-ae61-0ddbebc55c5d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:56:57.936Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e4b7ba0b-a63d-4c92-804a-484f1ee56980","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:56:57.936Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e4b7ba0b-a63d-4c92-804a-484f1ee56980","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:56:57.936Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e4b7ba0b-a63d-4c92-804a-484f1ee56980","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:56:57.936Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e4b7ba0b-a63d-4c92-804a-484f1ee56980"}
{"level":"info","ts":"2026-09-18T09:56:57.936Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e4b7ba0b-a63d-4c92-804a-484f1ee56980","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:56:57.936Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e4b7ba0b-a63d-4c92-804a-484f1ee56980","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:00.936Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"42e85784-415e-4220-bfd9-fd80f3f303b4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:00.936Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"42e85784-415e-4220-bfd9-fd80f3f303b4","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:00.936Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"42e85784-415e-4220-bfd9-fd80f3f303b4","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:00.936Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"42e85784-415e-4220-bfd9-fd80f3f303b4"}
{"level":"info","ts":"2026-09-18T09:57:00.937Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"42e85784-415e-4220-bfd9-fd80f3f303b4","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:57:00.937Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"42e85784-415e-4220-bfd9-fd80f3f303b4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:03.937Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"87e136c7-f39c-43c2-a3df-df05ed7e39e0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:03.937Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"87e136c7-f39c-43c2-a3df-df05ed7e39e0","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:03.937Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"87e136c7-f39c-43c2-a3df-df05ed7e39e0","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:03.937Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"87e136c7-f39c-43c2-a3df-df05ed7e39e0"}
{"level":"info","ts":"2026-09-18T09:57:03.937Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"87e136c7-f39c-43c2-a3df-df05ed7e39e0","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:57:03.937Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"87e136c7-f39c-43c2-a3df-df05ed7e39e0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:06.937Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"7a738ea2-0c38-48e8-a3d0-dff9ffacea66","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:06.937Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"7a738ea2-0c38-48e8-a3d0-dff9ffacea66","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:06.937Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"7a738ea2-0c38-48e8-a3d0-dff9ffacea66","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:06.938Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"7a738ea2-0c38-48e8-a3d0-dff9ffacea66"}
{"level":"info","ts":"2026-09-18T09:57:06.938Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"7a738ea2-0c38-48e8-a3d0-dff9ffacea66","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:57:06.938Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"7a738ea2-0c38-48e8-a3d0-dff9ffacea66","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:09.938Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"68f820a1-ab62-4126-b20d-ebc79e935c17","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:09.938Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"68f820a1-ab62-4126-b20d-ebc79e935c17","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:09.938Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"68f820a1-ab62-4126-b20d-ebc79e935c17","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:09.938Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"68f820a1-ab62-4126-b20d-ebc79e935c17"}
{"level":"info","ts":"2026-09-18T09:57:09.938Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"68f820a1-ab62-4126-b20d-ebc79e935c17","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:57:09.938Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"68f820a1-ab62-4126-b20d-ebc79e935c17","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:12.939Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"23ccf524-b9cf-4f58-9d63-7bf642a6bc0c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:12.939Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"23ccf524-b9cf-4f58-9d63-7bf642a6bc0c","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:12.940Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"23ccf524-b9cf-4f58-9d63-7bf642a6bc0c","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:12.940Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"23ccf524-b9cf-4f58-9d63-7bf642a6bc0c"}
{"level":"info","ts":"2026-09-18T09:57:12.940Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"23ccf524-b9cf-4f58-9d63-7bf642a6bc0c","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:57:12.940Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"23ccf524-b9cf-4f58-9d63-7bf642a6bc0c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:15.460Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"4cdadf06-ef78-4f4d-a15d-e042074bf543"}
{"level":"info","ts":"2026-09-18T09:57:15.460Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"4cdadf06-ef78-4f4d-a15d-e042074bf543"}
{"level":"info","ts":"2026-09-18T09:57:15.460Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"4cdadf06-ef78-4f4d-a15d-e042074bf543"}
{"level":"info","ts":"2026-09-18T09:57:15.460Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"4cdadf06-ef78-4f4d-a15d-e042074bf543","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:57:15.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"4cdadf06-ef78-4f4d-a15d-e042074bf543","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:57:15.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"4cdadf06-ef78-4f4d-a15d-e042074bf543","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:57:15.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"4cdadf06-ef78-4f4d-a15d-e042074bf543","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:57:15.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"4cdadf06-ef78-4f4d-a15d-e042074bf543","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:57:15.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"4cdadf06-ef78-4f4d-a15d-e042074bf543","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:57:15.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"4cdadf06-ef78-4f4d-a15d-e042074bf543","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:57:15.460Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"4cdadf06-ef78-4f4d-a15d-e042074bf543","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:57:15.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"4cdadf06-ef78-4f4d-a15d-e042074bf543","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:57:15.460Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"4cdadf06-ef78-4f4d-a15d-e042074bf543","podName":"fail-qvndg-small-group-worker-dktj6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:57:15.460Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"4cdadf06-ef78-4f4d-a15d-e042074bf543","podName":"fail-qvndg-small-group-worker-dktj6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:57:15.461Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"4cdadf06-ef78-4f4d-a15d-e042074bf543","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:57:15.461Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"4cdadf06-ef78-4f4d-a15d-e042074bf543","seconds":300}
{"level":"info","ts":"2026-09-18T09:57:15.941Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"c5b2fbb5-da14-41d8-b79e-0d4c73fb175c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:15.941Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"c5b2fbb5-da14-41d8-b79e-0d4c73fb175c","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:15.941Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"c5b2fbb5-da14-41d8-b79e-0d4c73fb175c","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:15.941Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"c5b2fbb5-da14-41d8-b79e-0d4c73fb175c"}
{"level":"info","ts":"2026-09-18T09:57:15.941Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"c5b2fbb5-da14-41d8-b79e-0d4c73fb175c","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:57:15.941Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"c5b2fbb5-da14-41d8-b79e-0d4c73fb175c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:16.465Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c93ab91c-4d30-4cc5-86e2-49434e5f61f6"}
{"level":"info","ts":"2026-09-18T09:57:16.465Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c93ab91c-4d30-4cc5-86e2-49434e5f61f6"}
{"level":"info","ts":"2026-09-18T09:57:16.465Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c93ab91c-4d30-4cc5-86e2-49434e5f61f6"}
{"level":"info","ts":"2026-09-18T09:57:16.465Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c93ab91c-4d30-4cc5-86e2-49434e5f61f6","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:57:16.465Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c93ab91c-4d30-4cc5-86e2-49434e5f61f6","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:57:16.465Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c93ab91c-4d30-4cc5-86e2-49434e5f61f6","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:57:16.465Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c93ab91c-4d30-4cc5-86e2-49434e5f61f6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:57:16.465Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c93ab91c-4d30-4cc5-86e2-49434e5f61f6","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:57:16.465Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c93ab91c-4d30-4cc5-86e2-49434e5f61f6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:57:16.465Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c93ab91c-4d30-4cc5-86e2-49434e5f61f6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:57:16.465Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c93ab91c-4d30-4cc5-86e2-49434e5f61f6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:57:16.465Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c93ab91c-4d30-4cc5-86e2-49434e5f61f6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:57:16.465Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c93ab91c-4d30-4cc5-86e2-49434e5f61f6","podName":"fail-qvndg-small-group-worker-dktj6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:57:16.465Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c93ab91c-4d30-4cc5-86e2-49434e5f61f6","podName":"fail-qvndg-small-group-worker-dktj6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:57:16.466Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c93ab91c-4d30-4cc5-86e2-49434e5f61f6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:57:16.466Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c93ab91c-4d30-4cc5-86e2-49434e5f61f6","seconds":300}
{"level":"info","ts":"2026-09-18T09:57:18.942Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"f1c68081-a186-44b9-9410-55d4ec9c8ad4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:18.942Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"f1c68081-a186-44b9-9410-55d4ec9c8ad4","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:18.942Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"f1c68081-a186-44b9-9410-55d4ec9c8ad4","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:18.942Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"f1c68081-a186-44b9-9410-55d4ec9c8ad4"}
{"level":"info","ts":"2026-09-18T09:57:18.942Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"f1c68081-a186-44b9-9410-55d4ec9c8ad4","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:57:18.942Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"f1c68081-a186-44b9-9410-55d4ec9c8ad4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:21.944Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e52e59cf-f5b7-42d6-99a0-33298fdc7ade","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:21.944Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e52e59cf-f5b7-42d6-99a0-33298fdc7ade","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:21.944Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e52e59cf-f5b7-42d6-99a0-33298fdc7ade","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:21.944Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e52e59cf-f5b7-42d6-99a0-33298fdc7ade"}
{"level":"info","ts":"2026-09-18T09:57:21.944Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e52e59cf-f5b7-42d6-99a0-33298fdc7ade","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:57:21.944Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"e52e59cf-f5b7-42d6-99a0-33298fdc7ade","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:24.945Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d824cd94-9a9f-45f9-82c5-97b0fe49ba42","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:24.945Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d824cd94-9a9f-45f9-82c5-97b0fe49ba42","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:24.945Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d824cd94-9a9f-45f9-82c5-97b0fe49ba42","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:24.945Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d824cd94-9a9f-45f9-82c5-97b0fe49ba42"}
{"level":"info","ts":"2026-09-18T09:57:24.945Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d824cd94-9a9f-45f9-82c5-97b0fe49ba42","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:57:24.945Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d824cd94-9a9f-45f9-82c5-97b0fe49ba42","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:25.514Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"46bdbc19-fffa-4485-b81a-9d130f3d2018"}
{"level":"info","ts":"2026-09-18T09:57:25.514Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"46bdbc19-fffa-4485-b81a-9d130f3d2018"}
{"level":"info","ts":"2026-09-18T09:57:25.514Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"46bdbc19-fffa-4485-b81a-9d130f3d2018"}
{"level":"info","ts":"2026-09-18T09:57:25.514Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"46bdbc19-fffa-4485-b81a-9d130f3d2018","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:57:25.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"46bdbc19-fffa-4485-b81a-9d130f3d2018","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:57:25.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"46bdbc19-fffa-4485-b81a-9d130f3d2018","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:57:25.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"46bdbc19-fffa-4485-b81a-9d130f3d2018","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:57:25.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"46bdbc19-fffa-4485-b81a-9d130f3d2018","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:57:25.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"46bdbc19-fffa-4485-b81a-9d130f3d2018","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:57:25.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"46bdbc19-fffa-4485-b81a-9d130f3d2018","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:57:25.515Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"46bdbc19-fffa-4485-b81a-9d130f3d2018","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:57:25.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"46bdbc19-fffa-4485-b81a-9d130f3d2018","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:57:25.515Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"46bdbc19-fffa-4485-b81a-9d130f3d2018","podName":"fail-qvndg-small-group-worker-dktj6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:57:25.515Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"46bdbc19-fffa-4485-b81a-9d130f3d2018","podName":"fail-qvndg-small-group-worker-dktj6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:57:25.515Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"46bdbc19-fffa-4485-b81a-9d130f3d2018","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:57:25.515Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"46bdbc19-fffa-4485-b81a-9d130f3d2018","seconds":300}
{"level":"info","ts":"2026-09-18T09:57:26.516Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c2c17801-9e90-49d1-8d28-6fa666666d70"}
{"level":"info","ts":"2026-09-18T09:57:26.516Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c2c17801-9e90-49d1-8d28-6fa666666d70"}
{"level":"info","ts":"2026-09-18T09:57:26.516Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c2c17801-9e90-49d1-8d28-6fa666666d70"}
{"level":"info","ts":"2026-09-18T09:57:26.516Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c2c17801-9e90-49d1-8d28-6fa666666d70","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:57:26.516Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c2c17801-9e90-49d1-8d28-6fa666666d70","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:57:26.516Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c2c17801-9e90-49d1-8d28-6fa666666d70","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:57:26.516Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c2c17801-9e90-49d1-8d28-6fa666666d70","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:57:26.516Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c2c17801-9e90-49d1-8d28-6fa666666d70","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:57:26.516Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c2c17801-9e90-49d1-8d28-6fa666666d70","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:57:26.516Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c2c17801-9e90-49d1-8d28-6fa666666d70","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:57:26.516Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c2c17801-9e90-49d1-8d28-6fa666666d70","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:57:26.516Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c2c17801-9e90-49d1-8d28-6fa666666d70","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:57:26.516Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c2c17801-9e90-49d1-8d28-6fa666666d70","podName":"fail-qvndg-small-group-worker-dktj6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:56:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:57:26.516Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c2c17801-9e90-49d1-8d28-6fa666666d70","podName":"fail-qvndg-small-group-worker-dktj6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:56:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:57:26.516Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"c2c17801-9e90-49d1-8d28-6fa666666d70","name":"fail-qvndg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:57:26.531Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"e1ce95ec-bb85-4683-95f7-d85298037210","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:26.531Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"ce6ed85e-fd80-4267-832b-138911467353","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:26.531Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"e1ce95ec-bb85-4683-95f7-d85298037210","pod":"fail-qvndg-small-group-worker-dktj6","ip":"10.132.0.39"}
{"level":"info","ts":"2026-09-18T09:57:26.531Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"e1ce95ec-bb85-4683-95f7-d85298037210","pod":"fail-qvndg-head-vlvq5","ip":"10.133.0.39"}
{"level":"info","ts":"2026-09-18T09:57:26.531Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"e1ce95ec-bb85-4683-95f7-d85298037210","rayCluster":"fail-qvndg","podIPs":["10.132.0.39","10.133.0.39"]}
{"level":"info","ts":"2026-09-18T09:57:26.531Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a0708e38-ec34-4ac0-892b-6bb071ec59ce","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:26.531Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a0708e38-ec34-4ac0-892b-6bb071ec59ce","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:26.531Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a0708e38-ec34-4ac0-892b-6bb071ec59ce","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:26.532Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a0708e38-ec34-4ac0-892b-6bb071ec59ce"}
{"level":"info","ts":"2026-09-18T09:57:26.532Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a0708e38-ec34-4ac0-892b-6bb071ec59ce","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:57:26.532Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a0708e38-ec34-4ac0-892b-6bb071ec59ce","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:56:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:26.532Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a0708e38-ec34-4ac0-892b-6bb071ec59ce","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:57:26.548Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3d1f8348-2720-456f-b3be-0fa16c30c8c9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:26.548Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3d1f8348-2720-456f-b3be-0fa16c30c8c9","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:26.548Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3d1f8348-2720-456f-b3be-0fa16c30c8c9","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:26.548Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3d1f8348-2720-456f-b3be-0fa16c30c8c9"}
{"level":"info","ts":"2026-09-18T09:57:26.548Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3d1f8348-2720-456f-b3be-0fa16c30c8c9","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:57:26.548Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3d1f8348-2720-456f-b3be-0fa16c30c8c9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:26.552Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"ce6ed85e-fd80-4267-832b-138911467353","name":"fail-qvndg-head"}
{"level":"info","ts":"2026-09-18T09:57:26.563Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"ce6ed85e-fd80-4267-832b-138911467353","name":"fail-qvndg-workers"}
{"level":"info","ts":"2026-09-18T09:57:26.563Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"ce6ed85e-fd80-4267-832b-138911467353","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:27.946Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"66e2d594-829c-45fa-b3c0-0c66de0e42c9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:27.946Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"66e2d594-829c-45fa-b3c0-0c66de0e42c9","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:27.946Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"66e2d594-829c-45fa-b3c0-0c66de0e42c9","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:27.946Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"66e2d594-829c-45fa-b3c0-0c66de0e42c9"}
{"level":"info","ts":"2026-09-18T09:57:27.946Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"66e2d594-829c-45fa-b3c0-0c66de0e42c9","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:57:27.946Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"66e2d594-829c-45fa-b3c0-0c66de0e42c9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:28.532Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"3817b14b-88f5-40e3-bf7e-e248bdc4f4b3"}
{"level":"info","ts":"2026-09-18T09:57:28.532Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"3817b14b-88f5-40e3-bf7e-e248bdc4f4b3"}
{"level":"info","ts":"2026-09-18T09:57:28.532Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"3817b14b-88f5-40e3-bf7e-e248bdc4f4b3"}
{"level":"info","ts":"2026-09-18T09:57:28.532Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"3817b14b-88f5-40e3-bf7e-e248bdc4f4b3","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:57:28.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"3817b14b-88f5-40e3-bf7e-e248bdc4f4b3","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:57:28.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"3817b14b-88f5-40e3-bf7e-e248bdc4f4b3","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:57:28.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"3817b14b-88f5-40e3-bf7e-e248bdc4f4b3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:57:28.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"3817b14b-88f5-40e3-bf7e-e248bdc4f4b3","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:57:28.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"3817b14b-88f5-40e3-bf7e-e248bdc4f4b3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:57:28.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"3817b14b-88f5-40e3-bf7e-e248bdc4f4b3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:57:28.532Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"3817b14b-88f5-40e3-bf7e-e248bdc4f4b3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:57:28.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"3817b14b-88f5-40e3-bf7e-e248bdc4f4b3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:57:28.532Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"3817b14b-88f5-40e3-bf7e-e248bdc4f4b3","podName":"fail-qvndg-small-group-worker-dktj6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:56:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:57:28.532Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"3817b14b-88f5-40e3-bf7e-e248bdc4f4b3","podName":"fail-qvndg-small-group-worker-dktj6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:56:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:57:28.532Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"3817b14b-88f5-40e3-bf7e-e248bdc4f4b3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:57:28.532Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"3817b14b-88f5-40e3-bf7e-e248bdc4f4b3","seconds":300}
{"level":"info","ts":"2026-09-18T09:57:30.947Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"717994dc-f8d2-4c77-aa8d-11cb8c3abd4c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:30.947Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"717994dc-f8d2-4c77-aa8d-11cb8c3abd4c","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:30.947Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"717994dc-f8d2-4c77-aa8d-11cb8c3abd4c","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:30.947Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"717994dc-f8d2-4c77-aa8d-11cb8c3abd4c"}
{"level":"info","ts":"2026-09-18T09:57:30.947Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"717994dc-f8d2-4c77-aa8d-11cb8c3abd4c","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:57:30.947Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"717994dc-f8d2-4c77-aa8d-11cb8c3abd4c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:31.080Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"f19265a0-3282-4d0c-8c1e-3386a4e40982","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:31.080Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"f19265a0-3282-4d0c-8c1e-3386a4e40982","pod":"fail-qvndg-head-vlvq5","ip":"10.133.0.39"}
{"level":"info","ts":"2026-09-18T09:57:31.080Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"f19265a0-3282-4d0c-8c1e-3386a4e40982","pod":"fail-qvndg-small-group-worker-dktj6","ip":"10.132.0.39"}
{"level":"info","ts":"2026-09-18T09:57:31.080Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"f19265a0-3282-4d0c-8c1e-3386a4e40982","rayCluster":"fail-qvndg","podIPs":["10.133.0.39","10.132.0.39"]}
{"level":"info","ts":"2026-09-18T09:57:33.948Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"46fc6f6a-94b2-43c9-bb5a-9cc5fbbc82e7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:33.948Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"46fc6f6a-94b2-43c9-bb5a-9cc5fbbc82e7","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:33.948Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"46fc6f6a-94b2-43c9-bb5a-9cc5fbbc82e7","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:33.948Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"46fc6f6a-94b2-43c9-bb5a-9cc5fbbc82e7"}
{"level":"info","ts":"2026-09-18T09:57:33.948Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"46fc6f6a-94b2-43c9-bb5a-9cc5fbbc82e7","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:57:33.948Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"46fc6f6a-94b2-43c9-bb5a-9cc5fbbc82e7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:36.949Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dddb61ce-2a26-41d0-b89d-0ce8ce2d0dab","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:36.949Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dddb61ce-2a26-41d0-b89d-0ce8ce2d0dab","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:36.949Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dddb61ce-2a26-41d0-b89d-0ce8ce2d0dab","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:36.949Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dddb61ce-2a26-41d0-b89d-0ce8ce2d0dab"}
{"level":"info","ts":"2026-09-18T09:57:36.949Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dddb61ce-2a26-41d0-b89d-0ce8ce2d0dab","RayCluster":"fail-qvndg","State":""}
{"level":"info","ts":"2026-09-18T09:57:36.949Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dddb61ce-2a26-41d0-b89d-0ce8ce2d0dab","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:37.543Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0c57f248-b928-4213-8946-124eb1c37512"}
{"level":"info","ts":"2026-09-18T09:57:37.543Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0c57f248-b928-4213-8946-124eb1c37512"}
{"level":"info","ts":"2026-09-18T09:57:37.543Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0c57f248-b928-4213-8946-124eb1c37512"}
{"level":"info","ts":"2026-09-18T09:57:37.543Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0c57f248-b928-4213-8946-124eb1c37512","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:57:37.543Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0c57f248-b928-4213-8946-124eb1c37512","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:57:37.543Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0c57f248-b928-4213-8946-124eb1c37512","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:57:37.543Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0c57f248-b928-4213-8946-124eb1c37512","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:57:37.543Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0c57f248-b928-4213-8946-124eb1c37512","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:57:37.543Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0c57f248-b928-4213-8946-124eb1c37512","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:57:37.543Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0c57f248-b928-4213-8946-124eb1c37512","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:57:37.543Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0c57f248-b928-4213-8946-124eb1c37512","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:57:37.543Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0c57f248-b928-4213-8946-124eb1c37512","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:57:37.543Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"0c57f248-b928-4213-8946-124eb1c37512","name":"fail-qvndg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:57:37.559Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"80b7535c-0332-4e9f-9cab-ba2a274c067e","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:37.559Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"70c09dae-3b80-48f3-8f23-a4b07ec312f4","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:37.559Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"80b7535c-0332-4e9f-9cab-ba2a274c067e","pod":"fail-qvndg-head-vlvq5","ip":"10.133.0.39"}
{"level":"info","ts":"2026-09-18T09:57:37.559Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"80b7535c-0332-4e9f-9cab-ba2a274c067e","pod":"fail-qvndg-small-group-worker-dktj6","ip":"10.132.0.39"}
{"level":"info","ts":"2026-09-18T09:57:37.559Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"80b7535c-0332-4e9f-9cab-ba2a274c067e","rayCluster":"fail-qvndg","podIPs":["10.133.0.39","10.132.0.39"]}
{"level":"info","ts":"2026-09-18T09:57:37.559Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a457d4e3-a744-4296-9d62-1db8b00c5f95","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:37.559Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a457d4e3-a744-4296-9d62-1db8b00c5f95","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:37.559Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a457d4e3-a744-4296-9d62-1db8b00c5f95","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:37.560Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a457d4e3-a744-4296-9d62-1db8b00c5f95"}
{"level":"info","ts":"2026-09-18T09:57:37.575Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a457d4e3-a744-4296-9d62-1db8b00c5f95","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-09-18T09:57:37.575Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a457d4e3-a744-4296-9d62-1db8b00c5f95","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:37.575Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a457d4e3-a744-4296-9d62-1db8b00c5f95","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:57:37.580Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"70c09dae-3b80-48f3-8f23-a4b07ec312f4","name":"fail-qvndg-head"}
{"level":"info","ts":"2026-09-18T09:57:37.588Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"85714a72-ec7a-4f70-850c-7301bf1a01c0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:37.588Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"85714a72-ec7a-4f70-850c-7301bf1a01c0","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:37.588Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"85714a72-ec7a-4f70-850c-7301bf1a01c0","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:37.588Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"85714a72-ec7a-4f70-850c-7301bf1a01c0"}
{"level":"info","ts":"2026-09-18T09:57:37.588Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"85714a72-ec7a-4f70-850c-7301bf1a01c0","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-09-18T09:57:37.588Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"85714a72-ec7a-4f70-850c-7301bf1a01c0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:37.588Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"85714a72-ec7a-4f70-850c-7301bf1a01c0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:57:37.595Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"70c09dae-3b80-48f3-8f23-a4b07ec312f4","name":"fail-qvndg-workers"}
{"level":"info","ts":"2026-09-18T09:57:37.595Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"70c09dae-3b80-48f3-8f23-a4b07ec312f4","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:37.599Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"85714a72-ec7a-4f70-850c-7301bf1a01c0","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T09:57:37.599Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"85714a72-ec7a-4f70-850c-7301bf1a01c0"}
{"level":"error","ts":"2026-09-18T09:57:37.599Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"85714a72-ec7a-4f70-850c-7301bf1a01c0","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:37.599Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dad98a98-32bd-48be-9ebb-9f5c34102c6d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:37.599Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dad98a98-32bd-48be-9ebb-9f5c34102c6d","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:37.599Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dad98a98-32bd-48be-9ebb-9f5c34102c6d","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:37.600Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dad98a98-32bd-48be-9ebb-9f5c34102c6d"}
{"level":"info","ts":"2026-09-18T09:57:37.600Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dad98a98-32bd-48be-9ebb-9f5c34102c6d","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:57:37.617Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dad98a98-32bd-48be-9ebb-9f5c34102c6d","JobId":"fail-m5xgt","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:37.617Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dad98a98-32bd-48be-9ebb-9f5c34102c6d"}
{"level":"error","ts":"2026-09-18T09:57:37.617Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"dad98a98-32bd-48be-9ebb-9f5c34102c6d","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:37.617Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"aca44f43-f73d-40c1-a9c1-15e23495ba3a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:37.617Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"aca44f43-f73d-40c1-a9c1-15e23495ba3a","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:37.617Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"aca44f43-f73d-40c1-a9c1-15e23495ba3a","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:37.617Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"aca44f43-f73d-40c1-a9c1-15e23495ba3a"}
{"level":"info","ts":"2026-09-18T09:57:37.617Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"aca44f43-f73d-40c1-a9c1-15e23495ba3a","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:57:37.622Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"aca44f43-f73d-40c1-a9c1-15e23495ba3a","JobId":"fail-m5xgt","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:37.622Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"aca44f43-f73d-40c1-a9c1-15e23495ba3a"}
{"level":"error","ts":"2026-09-18T09:57:37.622Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"aca44f43-f73d-40c1-a9c1-15e23495ba3a","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:37.628Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"98cb58ff-7a70-4a7d-9911-bff6c364e0c5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:37.628Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"98cb58ff-7a70-4a7d-9911-bff6c364e0c5","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:37.628Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"98cb58ff-7a70-4a7d-9911-bff6c364e0c5","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:37.628Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"98cb58ff-7a70-4a7d-9911-bff6c364e0c5"}
{"level":"info","ts":"2026-09-18T09:57:37.628Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"98cb58ff-7a70-4a7d-9911-bff6c364e0c5","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:57:37.669Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"98cb58ff-7a70-4a7d-9911-bff6c364e0c5","JobId":"fail-m5xgt","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:37.669Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"98cb58ff-7a70-4a7d-9911-bff6c364e0c5"}
{"level":"error","ts":"2026-09-18T09:57:37.669Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"98cb58ff-7a70-4a7d-9911-bff6c364e0c5","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:37.709Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"cefdcbce-d587-4e3e-b96f-4b5e3aa53184","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:37.709Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"cefdcbce-d587-4e3e-b96f-4b5e3aa53184","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:37.709Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"cefdcbce-d587-4e3e-b96f-4b5e3aa53184","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:37.709Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"cefdcbce-d587-4e3e-b96f-4b5e3aa53184"}
{"level":"info","ts":"2026-09-18T09:57:37.710Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"cefdcbce-d587-4e3e-b96f-4b5e3aa53184","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:57:37.714Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"cefdcbce-d587-4e3e-b96f-4b5e3aa53184","JobId":"fail-m5xgt","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:37.714Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"cefdcbce-d587-4e3e-b96f-4b5e3aa53184"}
{"level":"error","ts":"2026-09-18T09:57:37.714Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"cefdcbce-d587-4e3e-b96f-4b5e3aa53184","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:37.795Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"4738eb6f-ff43-47b9-a932-d224ce1b5682","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:37.795Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"4738eb6f-ff43-47b9-a932-d224ce1b5682","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:37.795Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"4738eb6f-ff43-47b9-a932-d224ce1b5682","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:37.795Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"4738eb6f-ff43-47b9-a932-d224ce1b5682"}
{"level":"info","ts":"2026-09-18T09:57:37.795Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"4738eb6f-ff43-47b9-a932-d224ce1b5682","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:57:37.800Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"4738eb6f-ff43-47b9-a932-d224ce1b5682","JobId":"fail-m5xgt","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:37.800Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"4738eb6f-ff43-47b9-a932-d224ce1b5682"}
{"level":"error","ts":"2026-09-18T09:57:37.800Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"4738eb6f-ff43-47b9-a932-d224ce1b5682","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:37.960Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"0e2300d4-d6ea-4f0c-86fb-1e93071b8d2e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:37.960Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"0e2300d4-d6ea-4f0c-86fb-1e93071b8d2e","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:37.960Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"0e2300d4-d6ea-4f0c-86fb-1e93071b8d2e","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:37.960Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"0e2300d4-d6ea-4f0c-86fb-1e93071b8d2e"}
{"level":"info","ts":"2026-09-18T09:57:37.960Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"0e2300d4-d6ea-4f0c-86fb-1e93071b8d2e","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:57:37.965Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"0e2300d4-d6ea-4f0c-86fb-1e93071b8d2e","JobId":"fail-m5xgt","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:37.965Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"0e2300d4-d6ea-4f0c-86fb-1e93071b8d2e"}
{"level":"error","ts":"2026-09-18T09:57:37.965Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"0e2300d4-d6ea-4f0c-86fb-1e93071b8d2e","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:38.286Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"05cadc0d-c44c-43ef-93e1-bdc48ff44837","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:38.286Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"05cadc0d-c44c-43ef-93e1-bdc48ff44837","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:38.286Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"05cadc0d-c44c-43ef-93e1-bdc48ff44837","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:38.286Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"05cadc0d-c44c-43ef-93e1-bdc48ff44837"}
{"level":"info","ts":"2026-09-18T09:57:38.286Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"05cadc0d-c44c-43ef-93e1-bdc48ff44837","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:57:38.291Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"05cadc0d-c44c-43ef-93e1-bdc48ff44837","JobId":"fail-m5xgt","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:38.291Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"05cadc0d-c44c-43ef-93e1-bdc48ff44837"}
{"level":"error","ts":"2026-09-18T09:57:38.291Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"05cadc0d-c44c-43ef-93e1-bdc48ff44837","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:38.597Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"4a5be9d8-e3d9-4ed8-beab-daf3803ebdf7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:38.597Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"4a5be9d8-e3d9-4ed8-beab-daf3803ebdf7","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:38.597Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"4a5be9d8-e3d9-4ed8-beab-daf3803ebdf7","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:38.597Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"4a5be9d8-e3d9-4ed8-beab-daf3803ebdf7"}
{"level":"info","ts":"2026-09-18T09:57:38.597Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"4a5be9d8-e3d9-4ed8-beab-daf3803ebdf7","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:57:38.602Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"4a5be9d8-e3d9-4ed8-beab-daf3803ebdf7","JobId":"fail-m5xgt","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:38.602Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"4a5be9d8-e3d9-4ed8-beab-daf3803ebdf7"}
{"level":"error","ts":"2026-09-18T09:57:38.602Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"4a5be9d8-e3d9-4ed8-beab-daf3803ebdf7","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:38.932Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b140721f-67b8-41f4-9945-9f2e2b913c0e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:38.932Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b140721f-67b8-41f4-9945-9f2e2b913c0e","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:38.932Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b140721f-67b8-41f4-9945-9f2e2b913c0e","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:38.932Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b140721f-67b8-41f4-9945-9f2e2b913c0e"}
{"level":"info","ts":"2026-09-18T09:57:38.932Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b140721f-67b8-41f4-9945-9f2e2b913c0e","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:57:38.937Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b140721f-67b8-41f4-9945-9f2e2b913c0e","JobId":"fail-m5xgt","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:38.937Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b140721f-67b8-41f4-9945-9f2e2b913c0e"}
{"level":"error","ts":"2026-09-18T09:57:38.937Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"b140721f-67b8-41f4-9945-9f2e2b913c0e","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:39.560Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f0849491-2b27-4a6a-b416-7ae1424354e7"}
{"level":"info","ts":"2026-09-18T09:57:39.560Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f0849491-2b27-4a6a-b416-7ae1424354e7"}
{"level":"info","ts":"2026-09-18T09:57:39.560Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f0849491-2b27-4a6a-b416-7ae1424354e7"}
{"level":"info","ts":"2026-09-18T09:57:39.560Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f0849491-2b27-4a6a-b416-7ae1424354e7","1 head service found":"fail-qvndg-head-svc"}
{"level":"info","ts":"2026-09-18T09:57:39.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f0849491-2b27-4a6a-b416-7ae1424354e7","Found 1 head Pod":"fail-qvndg-head-vlvq5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:57:39.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f0849491-2b27-4a6a-b416-7ae1424354e7","head Pod":"fail-qvndg-head-vlvq5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-qvndg-head-vlvq5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:57:39.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f0849491-2b27-4a6a-b416-7ae1424354e7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:57:39.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f0849491-2b27-4a6a-b416-7ae1424354e7","worker Pod":"fail-qvndg-small-group-worker-dktj6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-qvndg-small-group-worker-dktj6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:57:39.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f0849491-2b27-4a6a-b416-7ae1424354e7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:57:39.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f0849491-2b27-4a6a-b416-7ae1424354e7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:57:39.560Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f0849491-2b27-4a6a-b416-7ae1424354e7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:57:39.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f0849491-2b27-4a6a-b416-7ae1424354e7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:57:39.560Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f0849491-2b27-4a6a-b416-7ae1424354e7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:57:39.560Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"f0849491-2b27-4a6a-b416-7ae1424354e7","seconds":300}
{"level":"info","ts":"2026-09-18T09:57:41.498Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3058c4bf-d847-4813-9aca-ea80953a3630","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:41.498Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3058c4bf-d847-4813-9aca-ea80953a3630","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:41.498Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3058c4bf-d847-4813-9aca-ea80953a3630","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:41.498Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3058c4bf-d847-4813-9aca-ea80953a3630"}
{"level":"info","ts":"2026-09-18T09:57:41.498Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3058c4bf-d847-4813-9aca-ea80953a3630","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:57:41.503Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3058c4bf-d847-4813-9aca-ea80953a3630","JobId":"fail-m5xgt","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:41.503Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3058c4bf-d847-4813-9aca-ea80953a3630"}
{"level":"error","ts":"2026-09-18T09:57:41.503Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"3058c4bf-d847-4813-9aca-ea80953a3630","error":"Job fail-m5xgt does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:46.623Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"8f729d3b-17ec-41ae-b3bd-069563c63fbe","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:46.624Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"8f729d3b-17ec-41ae-b3bd-069563c63fbe","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:46.624Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"8f729d3b-17ec-41ae-b3bd-069563c63fbe","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:46.624Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"8f729d3b-17ec-41ae-b3bd-069563c63fbe"}
{"level":"info","ts":"2026-09-18T09:57:46.624Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"8f729d3b-17ec-41ae-b3bd-069563c63fbe","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:57:46.629Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"8f729d3b-17ec-41ae-b3bd-069563c63fbe","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:46.629Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"8f729d3b-17ec-41ae-b3bd-069563c63fbe","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:57:46.657Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a46ff678-1c04-44fb-9cf5-f4c738eddef7","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:46.657Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a46ff678-1c04-44fb-9cf5-f4c738eddef7","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:46.657Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a46ff678-1c04-44fb-9cf5-f4c738eddef7","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:46.658Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a46ff678-1c04-44fb-9cf5-f4c738eddef7"}
{"level":"info","ts":"2026-09-18T09:57:46.658Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a46ff678-1c04-44fb-9cf5-f4c738eddef7","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:57:46.660Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"a46ff678-1c04-44fb-9cf5-f4c738eddef7","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:49.658Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"1b9e0f97-cec7-4368-bdfc-c7781abcba17","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:49.658Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"1b9e0f97-cec7-4368-bdfc-c7781abcba17","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:49.658Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"1b9e0f97-cec7-4368-bdfc-c7781abcba17","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:49.658Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"1b9e0f97-cec7-4368-bdfc-c7781abcba17"}
{"level":"info","ts":"2026-09-18T09:57:49.658Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"1b9e0f97-cec7-4368-bdfc-c7781abcba17","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:57:49.661Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"1b9e0f97-cec7-4368-bdfc-c7781abcba17","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z","endTime":"2026-09-18T09:57:47Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:57:46,141\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-m5xgt: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:49.661Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"1b9e0f97-cec7-4368-bdfc-c7781abcba17","old JobStatus":"PENDING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:57:49.678Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"470454a2-b81c-4458-a350-ad52c0bec70f","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:49.678Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"470454a2-b81c-4458-a350-ad52c0bec70f","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:49.678Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"470454a2-b81c-4458-a350-ad52c0bec70f","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:49.678Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"470454a2-b81c-4458-a350-ad52c0bec70f"}
{"level":"info","ts":"2026-09-18T09:57:49.678Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"470454a2-b81c-4458-a350-ad52c0bec70f","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:57:49.680Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"470454a2-b81c-4458-a350-ad52c0bec70f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z","endTime":"2026-09-18T09:57:47Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:57:46,141\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-m5xgt: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z","endTime":"2026-09-18T09:57:47Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:57:46,141\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-m5xgt: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:52.678Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"87371d9d-54e9-453c-b881-0ee031c19328","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:52.678Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"87371d9d-54e9-453c-b881-0ee031c19328","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:52.678Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"87371d9d-54e9-453c-b881-0ee031c19328","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:52.679Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"87371d9d-54e9-453c-b881-0ee031c19328"}
{"level":"info","ts":"2026-09-18T09:57:52.679Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"87371d9d-54e9-453c-b881-0ee031c19328","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:57:52.681Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"87371d9d-54e9-453c-b881-0ee031c19328","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z","endTime":"2026-09-18T09:57:47Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:57:46,141\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-m5xgt: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z","endTime":"2026-09-18T09:57:47Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:57:46,141\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-m5xgt: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:55.683Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"15078eea-b9cf-4fa2-a936-efd41ddcfe74","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:55.683Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"15078eea-b9cf-4fa2-a936-efd41ddcfe74","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:55.683Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"15078eea-b9cf-4fa2-a936-efd41ddcfe74","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:55.683Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"15078eea-b9cf-4fa2-a936-efd41ddcfe74"}
{"level":"info","ts":"2026-09-18T09:57:55.683Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"15078eea-b9cf-4fa2-a936-efd41ddcfe74","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:57:55.686Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"15078eea-b9cf-4fa2-a936-efd41ddcfe74","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z","endTime":"2026-09-18T09:57:47Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:57:46,141\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-m5xgt: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z","endTime":"2026-09-18T09:57:47Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:57:46,141\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-m5xgt: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:57.678Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"79a60c85-4d9e-4288-9449-2ceae4d680b6","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:57.678Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"79a60c85-4d9e-4288-9449-2ceae4d680b6","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:57.678Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"79a60c85-4d9e-4288-9449-2ceae4d680b6","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:57.678Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"79a60c85-4d9e-4288-9449-2ceae4d680b6"}
{"level":"info","ts":"2026-09-18T09:57:57.678Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"79a60c85-4d9e-4288-9449-2ceae4d680b6","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:57:57.681Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"79a60c85-4d9e-4288-9449-2ceae4d680b6","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z","endTime":"2026-09-18T09:57:47Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:57:46,141\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-m5xgt: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z","endTime":"2026-09-18T09:57:47Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:57:46,141\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-m5xgt: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:58.685Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"1fcda16b-0a38-4ddb-82ec-cb40e354510e","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:58.686Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"1fcda16b-0a38-4ddb-82ec-cb40e354510e","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:58.686Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"1fcda16b-0a38-4ddb-82ec-cb40e354510e","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:58.686Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"1fcda16b-0a38-4ddb-82ec-cb40e354510e"}
{"level":"info","ts":"2026-09-18T09:57:58.686Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"1fcda16b-0a38-4ddb-82ec-cb40e354510e","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:57:58.689Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"1fcda16b-0a38-4ddb-82ec-cb40e354510e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z","endTime":"2026-09-18T09:57:47Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:57:46,141\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-m5xgt: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z","endTime":"2026-09-18T09:57:47Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:57:46,141\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-m5xgt: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:58.689Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d69d38ee-f4dc-4f88-bd73-3c0d17a1c769","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:58.689Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d69d38ee-f4dc-4f88-bd73-3c0d17a1c769","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:58.689Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d69d38ee-f4dc-4f88-bd73-3c0d17a1c769","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:58.689Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d69d38ee-f4dc-4f88-bd73-3c0d17a1c769"}
{"level":"info","ts":"2026-09-18T09:57:58.689Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d69d38ee-f4dc-4f88-bd73-3c0d17a1c769","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:57:58.691Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d69d38ee-f4dc-4f88-bd73-3c0d17a1c769","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z","endTime":"2026-09-18T09:57:47Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:57:46,141\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-m5xgt: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z","endTime":"2026-09-18T09:57:47Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:57:46,141\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-m5xgt: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:58.700Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d0d02bae-faa2-4323-be7b-9e47a61b3ea5","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:58.700Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d0d02bae-faa2-4323-be7b-9e47a61b3ea5","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:58.700Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d0d02bae-faa2-4323-be7b-9e47a61b3ea5","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:58.700Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d0d02bae-faa2-4323-be7b-9e47a61b3ea5"}
{"level":"info","ts":"2026-09-18T09:57:58.700Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d0d02bae-faa2-4323-be7b-9e47a61b3ea5","rayCluster":"fail-qvndg","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T09:57:58.702Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d0d02bae-faa2-4323-be7b-9e47a61b3ea5","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z","endTime":"2026-09-18T09:57:47Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:57:46,141\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-m5xgt: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T09:57:46Z","endTime":"2026-09-18T09:57:47Z"},"jobId":"fail-m5xgt","rayClusterName":"fail-qvndg","dashboardURL":"fail-qvndg-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"AppFailed","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-09-18 09:57:46,141\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-m5xgt: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-09-18T09:56:00Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:57:37Z","stateTransitionTimes":{"ready":"2026-09-18T09:57:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.39","serviceIP":"10.133.0.39","podName":"fail-qvndg-head-vlvq5","serviceName":"fail-qvndg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:56:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:57:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:56:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:56:01Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:57:58.702Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"d0d02bae-faa2-4323-be7b-9e47a61b3ea5","old JobStatus":"FAILED","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-09-18T09:57:58.717Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"650db8cb-c6f8-4677-bd8b-0958870c153c","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:57:59.686Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"32aa8f1e-0d32-4456-81eb-9436f6c11ad6","DeletionTimestamp":"2026-09-18 09:57:59 +0000 UTC"}
{"level":"info","ts":"2026-09-18T09:57:59.686Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"32aa8f1e-0d32-4456-81eb-9436f6c11ad6","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:57:59.701Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"9d375a97-9b7d-4d1b-94c1-151504818e5f"}
{"level":"info","ts":"2026-09-18T09:57:59.712Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"033ee8c4-7e25-4ebd-8cc9-b594cb31d7bb"}
{"level":"info","ts":"2026-09-18T09:57:59.714Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"f365bd39-c468-41ad-ae99-2cbf70db318d"}
{"level":"info","ts":"2026-09-18T09:57:59.719Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"53615392-3df6-4cdd-9415-acd90ce07f42"}
{"level":"info","ts":"2026-09-18T09:57:59.719Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"dc545a47-7a6e-44d2-8143-439a67b53bcf","rayCluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:59.719Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"reconcileID":"53506aa6-d8cc-4be8-9a3d-2d67fe95231d"}
{"level":"info","ts":"2026-09-18T09:57:59.719Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"dc545a47-7a6e-44d2-8143-439a67b53bcf","pod":"fail-qvndg-head-vlvq5","ip":"10.133.0.39"}
{"level":"info","ts":"2026-09-18T09:57:59.719Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"dc545a47-7a6e-44d2-8143-439a67b53bcf","pod":"fail-qvndg-small-group-worker-dktj6","ip":"10.132.0.39"}
{"level":"info","ts":"2026-09-18T09:57:59.719Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"6aaacd9a-b9ea-4c7a-b7f7-138b397e3de5"}
{"level":"info","ts":"2026-09-18T09:57:59.719Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"dc545a47-7a6e-44d2-8143-439a67b53bcf","rayCluster":"fail-qvndg","podIPs":["10.133.0.39","10.132.0.39"]}
{"level":"info","ts":"2026-09-18T09:57:59.719Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"bff0991e-0b49-45a2-bea5-1687b57b93a8","namespacedName":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:59.719Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"bff0991e-0b49-45a2-bea5-1687b57b93a8","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:59.719Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"bff0991e-0b49-45a2-bea5-1687b57b93a8","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:59.719Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"bff0991e-0b49-45a2-bea5-1687b57b93a8","namespace":"test-ns-ltx56","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T09:57:59.730Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"bff0991e-0b49-45a2-bea5-1687b57b93a8","name":"kuberay-gateway-access","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T09:57:59.737Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"bff0991e-0b49-45a2-bea5-1687b57b93a8","name":"test-ns-ltx56-fail-qvndg","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T09:57:59.742Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"bff0991e-0b49-45a2-bea5-1687b57b93a8","configMap":"kube-rbac-proxy-config-fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:59.747Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"bff0991e-0b49-45a2-bea5-1687b57b93a8","serviceAccount":"fail-qvndg-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:57:59.768Z","logger":"raycluster-resource","msg":"default","name":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:59.768Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-qvndg","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T09:57:59.778Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"bff0991e-0b49-45a2-bea5-1687b57b93a8","cluster":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:59.778Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"6b254c5c-1dee-4380-b5a8-cc010544cb6e","namespacedName":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:59.779Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"6b254c5c-1dee-4380-b5a8-cc010544cb6e"}
{"level":"info","ts":"2026-09-18T09:57:59.779Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"6b254c5c-1dee-4380-b5a8-cc010544cb6e","namespace":"test-ns-ltx56","authClusters":0}
{"level":"info","ts":"2026-09-18T09:57:59.779Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"6b254c5c-1dee-4380-b5a8-cc010544cb6e","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T09:57:59.779Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"917b2d70-f0ca-455a-958a-909b4c1bf4ca","namespacedName":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:59.779Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"917b2d70-f0ca-455a-958a-909b4c1bf4ca"}
{"level":"info","ts":"2026-09-18T09:57:59.779Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"917b2d70-f0ca-455a-958a-909b4c1bf4ca","namespace":"test-ns-ltx56","authClusters":0}
{"level":"info","ts":"2026-09-18T09:57:59.779Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"917b2d70-f0ca-455a-958a-909b4c1bf4ca","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T09:57:59.779Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"db150f75-270a-4e25-9c99-943d678f7f51","namespace":"test-ns-ltx56","clusterName":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:59.779Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"59c78e52-7e40-4dcf-a4f7-a0c3bb1d6d52"}
{"level":"info","ts":"2026-09-18T09:57:59.779Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"9b9fc338-457f-4d3b-af56-2c1fd51f0526"}
{"level":"info","ts":"2026-09-18T09:57:59.783Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"2875c8d8-4b8f-4647-8253-dac322da82f3"}
{"level":"error","ts":"2026-09-18T09:57:59.790Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"db150f75-270a-4e25-9c99-943d678f7f51","issuer":"ray-selfsigned-issuer-fail-qvndg","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-qvndg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:59.800Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"a9c9ce4a-fd12-46cb-928d-67f52dc2adad"}
{"level":"info","ts":"2026-09-18T09:57:59.800Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"5d2dbeab-3fc0-4f5d-901c-cc1288ddd07d","namespacedName":{"name":"fail-qvndg","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:57:59.800Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"5d2dbeab-3fc0-4f5d-901c-cc1288ddd07d"}
{"level":"info","ts":"2026-09-18T09:57:59.800Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"5d2dbeab-3fc0-4f5d-901c-cc1288ddd07d","namespace":"test-ns-ltx56","authClusters":0}
{"level":"info","ts":"2026-09-18T09:57:59.800Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"5d2dbeab-3fc0-4f5d-901c-cc1288ddd07d","namespace":"test-ns-ltx56"}
{"level":"error","ts":"2026-09-18T09:57:59.800Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"db150f75-270a-4e25-9c99-943d678f7f51","certificate":"ray-head-cert-fail-qvndg","error":"certificates.cert-manager.io \"ray-head-cert-fail-qvndg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:57:59.804Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"db150f75-270a-4e25-9c99-943d678f7f51","certificate":"ray-ca-certificate-fail-qvndg","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-qvndg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:59.828Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"db150f75-270a-4e25-9c99-943d678f7f51","errorCount":3}
{"level":"info","ts":"2026-09-18T09:57:59.828Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"db150f75-270a-4e25-9c99-943d678f7f51"}
{"level":"error","ts":"2026-09-18T09:57:59.828Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"db150f75-270a-4e25-9c99-943d678f7f51","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-qvndg\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:57:59.833Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"99622b7a-1462-4f31-b647-4cc6b783615f","namespace":"test-ns-ltx56","clusterName":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:57:59.833Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-qvndg","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-qvndg","reconcileID":"99622b7a-1462-4f31-b647-4cc6b783615f","namespace":"test-ns-ltx56","clusterName":"fail-qvndg"}
{"level":"info","ts":"2026-09-18T09:58:00.710Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"887a7915-623f-4040-ba46-1b2d7849f293","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:00.710Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"887a7915-623f-4040-ba46-1b2d7849f293","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T09:58:00.722Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"887a7915-623f-4040-ba46-1b2d7849f293"}
{"level":"info","ts":"2026-09-18T09:58:00.722Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"887a7915-623f-4040-ba46-1b2d7849f293","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T09:58:00.722Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"887a7915-623f-4040-ba46-1b2d7849f293","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:58:00.722Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"887a7915-623f-4040-ba46-1b2d7849f293","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:58:00.733Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5df14636-2767-4276-b2f7-f559e3ecb350","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:00.733Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5df14636-2767-4276-b2f7-f559e3ecb350","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.733Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5df14636-2767-4276-b2f7-f559e3ecb350","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:00.733Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5df14636-2767-4276-b2f7-f559e3ecb350","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:00.738Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.738Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T09:58:00.753Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5df14636-2767-4276-b2f7-f559e3ecb350","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5df14636-2767-4276-b2f7-f559e3ecb350"}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5df14636-2767-4276-b2f7-f559e3ecb350","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5df14636-2767-4276-b2f7-f559e3ecb350","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"e4e0ea81-b673-491b-b1ab-c1743c40663e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"e4e0ea81-b673-491b-b1ab-c1743c40663e","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"e4e0ea81-b673-491b-b1ab-c1743c40663e","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"78ff6711-4175-41c7-844b-62596156a14f"}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"78ff6711-4175-41c7-844b-62596156a14f"}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"78ff6711-4175-41c7-844b-62596156a14f"}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"e4e0ea81-b673-491b-b1ab-c1743c40663e"}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"e4e0ea81-b673-491b-b1ab-c1743c40663e","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"e4e0ea81-b673-491b-b1ab-c1743c40663e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"08186e6b-9e21-4afd-82fa-78d7ef2de011","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"1984ceca-e270-44df-8427-eddfa489da09","namespacedName":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"1984ceca-e270-44df-8427-eddfa489da09","mode":"IntegratedOAuth","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"1984ceca-e270-44df-8427-eddfa489da09","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"1984ceca-e270-44df-8427-eddfa489da09","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6df4c429-bca5-4c4c-b0e1-a63c319fdc77","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.754Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6df4c429-bca5-4c4c-b0e1-a63c319fdc77","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.760Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.760Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T09:58:00.766Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"78ff6711-4175-41c7-844b-62596156a14f","name":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:00.766Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"78ff6711-4175-41c7-844b-62596156a14f","error":"head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"error","ts":"2026-09-18T09:58:00.766Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"78ff6711-4175-41c7-844b-62596156a14f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.766Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"78ff6711-4175-41c7-844b-62596156a14f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"info","ts":"2026-09-18T09:58:00.766Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"78ff6711-4175-41c7-844b-62596156a14f","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:00.767Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"78ff6711-4175-41c7-844b-62596156a14f","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:00.767Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"78ff6711-4175-41c7-844b-62596156a14f","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:00.767Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"78ff6711-4175-41c7-844b-62596156a14f","name":"fail-k8s-job-q64ml","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:58:00.777Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"08186e6b-9e21-4afd-82fa-78d7ef2de011","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.777Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"9ef49b56-c4d3-4751-aa63-2ed8cf629fc3","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.778Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1f722a55-c193-4205-b3b2-a32f67736dd5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:00.778Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1f722a55-c193-4205-b3b2-a32f67736dd5","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.778Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1f722a55-c193-4205-b3b2-a32f67736dd5","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:00.778Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1f722a55-c193-4205-b3b2-a32f67736dd5"}
{"level":"info","ts":"2026-09-18T09:58:00.778Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1f722a55-c193-4205-b3b2-a32f67736dd5","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:00.778Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1f722a55-c193-4205-b3b2-a32f67736dd5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T09:58:00.780Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6df4c429-bca5-4c4c-b0e1-a63c319fdc77","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.780Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6df4c429-bca5-4c4c-b0e1-a63c319fdc77","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.781Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"78ff6711-4175-41c7-844b-62596156a14f","name":"fail-k8s-job-q64ml","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-q64ml\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml"}}
{"level":"info","ts":"2026-09-18T09:58:00.781Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"78ff6711-4175-41c7-844b-62596156a14f"}
{"level":"error","ts":"2026-09-18T09:58:00.781Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"78ff6711-4175-41c7-844b-62596156a14f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.782Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"320991e0-8e6b-456f-82eb-9b9f9f06c790"}
{"level":"info","ts":"2026-09-18T09:58:00.782Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"320991e0-8e6b-456f-82eb-9b9f9f06c790"}
{"level":"info","ts":"2026-09-18T09:58:00.782Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"320991e0-8e6b-456f-82eb-9b9f9f06c790"}
{"level":"info","ts":"2026-09-18T09:58:00.782Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"320991e0-8e6b-456f-82eb-9b9f9f06c790","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:00.782Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"320991e0-8e6b-456f-82eb-9b9f9f06c790","error":"head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"error","ts":"2026-09-18T09:58:00.782Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"320991e0-8e6b-456f-82eb-9b9f9f06c790","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.782Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"320991e0-8e6b-456f-82eb-9b9f9f06c790","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"info","ts":"2026-09-18T09:58:00.782Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"320991e0-8e6b-456f-82eb-9b9f9f06c790","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:00.782Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"320991e0-8e6b-456f-82eb-9b9f9f06c790","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:00.782Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"320991e0-8e6b-456f-82eb-9b9f9f06c790","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:00.782Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"320991e0-8e6b-456f-82eb-9b9f9f06c790","name":"fail-k8s-job-q64ml","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:58:00.783Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"1984ceca-e270-44df-8427-eddfa489da09","name":"fail-k8s-job-q64ml-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:58:00.788Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"1984ceca-e270-44df-8427-eddfa489da09","name":"kuberay-gateway-access","namespace":"test-ns-ltx56","operation":"created"}
{"level":"info","ts":"2026-09-18T09:58:00.791Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6df4c429-bca5-4c4c-b0e1-a63c319fdc77","rayCluster":"fail-k8s-job-q64ml","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T09:58:00.791Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6df4c429-bca5-4c4c-b0e1-a63c319fdc77","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.795Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"9ef49b56-c4d3-4751-aa63-2ed8cf629fc3","name":"fail-k8s-job-q64ml-head"}
{"level":"info","ts":"2026-09-18T09:58:00.796Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"320991e0-8e6b-456f-82eb-9b9f9f06c790"}
{"level":"error","ts":"2026-09-18T09:58:00.796Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"320991e0-8e6b-456f-82eb-9b9f9f06c790","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.796Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a410dd26-6252-4611-9d3b-9b3b63e480e3"}
{"level":"info","ts":"2026-09-18T09:58:00.796Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a410dd26-6252-4611-9d3b-9b3b63e480e3"}
{"level":"info","ts":"2026-09-18T09:58:00.796Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a410dd26-6252-4611-9d3b-9b3b63e480e3"}
{"level":"info","ts":"2026-09-18T09:58:00.796Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a410dd26-6252-4611-9d3b-9b3b63e480e3","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:00.796Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a410dd26-6252-4611-9d3b-9b3b63e480e3","error":"head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"error","ts":"2026-09-18T09:58:00.796Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a410dd26-6252-4611-9d3b-9b3b63e480e3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.796Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a410dd26-6252-4611-9d3b-9b3b63e480e3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"info","ts":"2026-09-18T09:58:00.796Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a410dd26-6252-4611-9d3b-9b3b63e480e3","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:00.796Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a410dd26-6252-4611-9d3b-9b3b63e480e3","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:00.796Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a410dd26-6252-4611-9d3b-9b3b63e480e3","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:00.796Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a410dd26-6252-4611-9d3b-9b3b63e480e3","name":"fail-k8s-job-q64ml","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:58:00.797Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"1984ceca-e270-44df-8427-eddfa489da09","name":"test-ns-ltx56-fail-k8s-job-q64ml","operation":"created"}
{"level":"info","ts":"2026-09-18T09:58:00.797Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"134e1c33-7d2c-4bd4-9ec7-461f6ae1b284","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:00.797Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"134e1c33-7d2c-4bd4-9ec7-461f6ae1b284","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.797Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"134e1c33-7d2c-4bd4-9ec7-461f6ae1b284","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:00.866Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"134e1c33-7d2c-4bd4-9ec7-461f6ae1b284"}
{"level":"info","ts":"2026-09-18T09:58:00.866Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"134e1c33-7d2c-4bd4-9ec7-461f6ae1b284","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:00.866Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"134e1c33-7d2c-4bd4-9ec7-461f6ae1b284","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:00.866Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"134e1c33-7d2c-4bd4-9ec7-461f6ae1b284","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:58:00.866Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6df4c429-bca5-4c4c-b0e1-a63c319fdc77","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.867Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"1984ceca-e270-44df-8427-eddfa489da09","name":"kube-rbac-proxy-config-fail-k8s-job-q64ml","operation":"created"}
{"level":"info","ts":"2026-09-18T09:58:00.871Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"9ef49b56-c4d3-4751-aa63-2ed8cf629fc3","name":"fail-k8s-job-q64ml-workers"}
{"level":"info","ts":"2026-09-18T09:58:00.871Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"9ef49b56-c4d3-4751-aa63-2ed8cf629fc3","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.872Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"e9e0dad0-1fd2-46ac-ae26-23bbb14f290c","cluster":"fail-k8s-job-q64ml"}
{"level":"error","ts":"2026-09-18T09:58:00.873Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"1984ceca-e270-44df-8427-eddfa489da09","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-q64ml\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.873Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"1984ceca-e270-44df-8427-eddfa489da09"}
{"level":"error","ts":"2026-09-18T09:58:00.873Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"1984ceca-e270-44df-8427-eddfa489da09","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-q64ml\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.873Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a410dd26-6252-4611-9d3b-9b3b63e480e3","name":"fail-k8s-job-q64ml","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-q64ml\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml"}}
{"level":"info","ts":"2026-09-18T09:58:00.873Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6c4a8e00-bb3f-4acd-8e65-b4acdbfa8d34","namespacedName":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:00.873Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a410dd26-6252-4611-9d3b-9b3b63e480e3"}
{"level":"info","ts":"2026-09-18T09:58:00.873Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6c4a8e00-bb3f-4acd-8e65-b4acdbfa8d34","mode":"IntegratedOAuth","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.873Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6c4a8e00-bb3f-4acd-8e65-b4acdbfa8d34","cluster":"fail-k8s-job-q64ml"}
{"level":"error","ts":"2026-09-18T09:58:00.873Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a410dd26-6252-4611-9d3b-9b3b63e480e3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.873Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6cb57fdd-4ecc-4737-b7b2-033a2dcbe15c"}
{"level":"info","ts":"2026-09-18T09:58:00.873Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6cb57fdd-4ecc-4737-b7b2-033a2dcbe15c"}
{"level":"info","ts":"2026-09-18T09:58:00.873Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6cb57fdd-4ecc-4737-b7b2-033a2dcbe15c"}
{"level":"info","ts":"2026-09-18T09:58:00.873Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6cb57fdd-4ecc-4737-b7b2-033a2dcbe15c","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:00.873Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6cb57fdd-4ecc-4737-b7b2-033a2dcbe15c","error":"head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"error","ts":"2026-09-18T09:58:00.873Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6cb57fdd-4ecc-4737-b7b2-033a2dcbe15c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.873Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6cb57fdd-4ecc-4737-b7b2-033a2dcbe15c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"info","ts":"2026-09-18T09:58:00.873Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6cb57fdd-4ecc-4737-b7b2-033a2dcbe15c","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:00.873Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6cb57fdd-4ecc-4737-b7b2-033a2dcbe15c","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:00.873Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6cb57fdd-4ecc-4737-b7b2-033a2dcbe15c","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:00.873Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6cb57fdd-4ecc-4737-b7b2-033a2dcbe15c"}
{"level":"error","ts":"2026-09-18T09:58:00.873Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6cb57fdd-4ecc-4737-b7b2-033a2dcbe15c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.879Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"58a28b0e-15ef-4c06-b226-a274874ea1d9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:00.879Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"58a28b0e-15ef-4c06-b226-a274874ea1d9","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.879Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"58a28b0e-15ef-4c06-b226-a274874ea1d9","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:00.879Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"58a28b0e-15ef-4c06-b226-a274874ea1d9"}
{"level":"info","ts":"2026-09-18T09:58:00.879Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"58a28b0e-15ef-4c06-b226-a274874ea1d9","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:00.879Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"58a28b0e-15ef-4c06-b226-a274874ea1d9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:00.880Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6df4c429-bca5-4c4c-b0e1-a63c319fdc77","rayCluster":"fail-k8s-job-q64ml","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:58:00.885Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6c4a8e00-bb3f-4acd-8e65-b4acdbfa8d34","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.885Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"5eda86df-a2e4-4919-910c-236213dd4709","namespacedName":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:00.885Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"5eda86df-a2e4-4919-910c-236213dd4709","mode":"IntegratedOAuth","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.885Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"5eda86df-a2e4-4919-910c-236213dd4709","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.966Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f7f5c8bd-999f-4eb8-b12a-0a7a3d374782","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:00.966Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f7f5c8bd-999f-4eb8-b12a-0a7a3d374782","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.966Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f7f5c8bd-999f-4eb8-b12a-0a7a3d374782","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:00.966Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"e9e0dad0-1fd2-46ac-ae26-23bbb14f290c","name":"fail-k8s-job-q64ml-head"}
{"level":"info","ts":"2026-09-18T09:58:00.966Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f7f5c8bd-999f-4eb8-b12a-0a7a3d374782"}
{"level":"info","ts":"2026-09-18T09:58:00.966Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f7f5c8bd-999f-4eb8-b12a-0a7a3d374782","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:00.966Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f7f5c8bd-999f-4eb8-b12a-0a7a3d374782","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:00.966Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f7f5c8bd-999f-4eb8-b12a-0a7a3d374782","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:58:00.966Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c0a282e0-e5f2-4cba-bb45-15b8d9f1ad8f"}
{"level":"info","ts":"2026-09-18T09:58:00.966Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c0a282e0-e5f2-4cba-bb45-15b8d9f1ad8f"}
{"level":"info","ts":"2026-09-18T09:58:00.966Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c0a282e0-e5f2-4cba-bb45-15b8d9f1ad8f"}
{"level":"info","ts":"2026-09-18T09:58:00.966Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c0a282e0-e5f2-4cba-bb45-15b8d9f1ad8f","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:00.966Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c0a282e0-e5f2-4cba-bb45-15b8d9f1ad8f","error":"head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"error","ts":"2026-09-18T09:58:00.966Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c0a282e0-e5f2-4cba-bb45-15b8d9f1ad8f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.967Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c0a282e0-e5f2-4cba-bb45-15b8d9f1ad8f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"info","ts":"2026-09-18T09:58:00.967Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c0a282e0-e5f2-4cba-bb45-15b8d9f1ad8f","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:00.967Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c0a282e0-e5f2-4cba-bb45-15b8d9f1ad8f","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:00.967Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c0a282e0-e5f2-4cba-bb45-15b8d9f1ad8f","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:00.967Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c0a282e0-e5f2-4cba-bb45-15b8d9f1ad8f"}
{"level":"error","ts":"2026-09-18T09:58:00.967Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c0a282e0-e5f2-4cba-bb45-15b8d9f1ad8f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.967Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6df4c429-bca5-4c4c-b0e1-a63c319fdc77","rayCluster":"fail-k8s-job-q64ml","podIPs":[]}
{"level":"error","ts":"2026-09-18T09:58:00.967Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6df4c429-bca5-4c4c-b0e1-a63c319fdc77","error":"Certificate.cert-manager.io \"ray-worker-cert-fail-k8s-job-q64ml\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.967Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6df4c429-bca5-4c4c-b0e1-a63c319fdc77"}
{"level":"error","ts":"2026-09-18T09:58:00.967Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6df4c429-bca5-4c4c-b0e1-a63c319fdc77","error":"Certificate.cert-manager.io \"ray-worker-cert-fail-k8s-job-q64ml\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.967Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6543a47d-a775-4b3d-9423-dd68d27f5264","rayCluster":"fail-k8s-job-q64ml"}
{"level":"error","ts":"2026-09-18T09:58:00.967Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"5eda86df-a2e4-4919-910c-236213dd4709","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-q64ml\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.967Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"5eda86df-a2e4-4919-910c-236213dd4709"}
{"level":"error","ts":"2026-09-18T09:58:00.967Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"5eda86df-a2e4-4919-910c-236213dd4709","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-q64ml\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.972Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"378846e3-3cd9-4b7e-af26-e8b03acaaf0e","namespacedName":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:00.972Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"378846e3-3cd9-4b7e-af26-e8b03acaaf0e","mode":"IntegratedOAuth","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.972Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"378846e3-3cd9-4b7e-af26-e8b03acaaf0e","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.978Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"e9e0dad0-1fd2-46ac-ae26-23bbb14f290c","name":"fail-k8s-job-q64ml-workers"}
{"level":"info","ts":"2026-09-18T09:58:00.978Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"e9e0dad0-1fd2-46ac-ae26-23bbb14f290c","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.978Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"122aa794-d5ea-4ea3-a78e-9577c6b1c403","cluster":"fail-k8s-job-q64ml"}
{"level":"error","ts":"2026-09-18T09:58:00.981Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6543a47d-a775-4b3d-9423-dd68d27f5264","error":"certificates.cert-manager.io \"ray-worker-cert-fail-k8s-job-q64ml\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T09:58:00.981Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6543a47d-a775-4b3d-9423-dd68d27f5264","error":"certificates.cert-manager.io \"ray-worker-cert-fail-k8s-job-q64ml\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.981Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6543a47d-a775-4b3d-9423-dd68d27f5264"}
{"level":"error","ts":"2026-09-18T09:58:00.981Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"6543a47d-a775-4b3d-9423-dd68d27f5264","error":"certificates.cert-manager.io \"ray-worker-cert-fail-k8s-job-q64ml\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:00.981Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"45e328b9-e181-4b76-822b-f22750301166","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.981Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"45e328b9-e181-4b76-822b-f22750301166"}
{"level":"info","ts":"2026-09-18T09:58:00.983Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d7ebde63-6bca-41fa-853f-61c30f334861","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:00.983Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d7ebde63-6bca-41fa-853f-61c30f334861","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.983Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d7ebde63-6bca-41fa-853f-61c30f334861","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:00.983Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d7ebde63-6bca-41fa-853f-61c30f334861"}
{"level":"info","ts":"2026-09-18T09:58:00.983Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d7ebde63-6bca-41fa-853f-61c30f334861","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:00.983Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d7ebde63-6bca-41fa-853f-61c30f334861","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:00.983Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"378846e3-3cd9-4b7e-af26-e8b03acaaf0e","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:00.990Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"122aa794-d5ea-4ea3-a78e-9577c6b1c403","name":"fail-k8s-job-q64ml-head"}
{"level":"info","ts":"2026-09-18T09:58:01.066Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a30796a6-da3e-44fd-9bc0-7454404aec1d"}
{"level":"info","ts":"2026-09-18T09:58:01.066Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a30796a6-da3e-44fd-9bc0-7454404aec1d"}
{"level":"info","ts":"2026-09-18T09:58:01.066Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a30796a6-da3e-44fd-9bc0-7454404aec1d"}
{"level":"info","ts":"2026-09-18T09:58:01.066Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a30796a6-da3e-44fd-9bc0-7454404aec1d","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:01.066Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a30796a6-da3e-44fd-9bc0-7454404aec1d","error":"head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"error","ts":"2026-09-18T09:58:01.066Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a30796a6-da3e-44fd-9bc0-7454404aec1d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:01.066Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a30796a6-da3e-44fd-9bc0-7454404aec1d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"info","ts":"2026-09-18T09:58:01.066Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a30796a6-da3e-44fd-9bc0-7454404aec1d","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:01.066Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a30796a6-da3e-44fd-9bc0-7454404aec1d","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:01.066Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a30796a6-da3e-44fd-9bc0-7454404aec1d","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:01.066Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a30796a6-da3e-44fd-9bc0-7454404aec1d"}
{"level":"error","ts":"2026-09-18T09:58:01.066Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a30796a6-da3e-44fd-9bc0-7454404aec1d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:01.066Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"e6637561-9887-4361-83df-da448a9594d1","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:01.066Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"e6637561-9887-4361-83df-da448a9594d1"}
{"level":"info","ts":"2026-09-18T09:58:01.071Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"122aa794-d5ea-4ea3-a78e-9577c6b1c403","name":"fail-k8s-job-q64ml-workers"}
{"level":"info","ts":"2026-09-18T09:58:01.071Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"122aa794-d5ea-4ea3-a78e-9577c6b1c403","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:01.227Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"09951a01-a5c0-4e24-9e51-70994d3b0bd0"}
{"level":"info","ts":"2026-09-18T09:58:01.227Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"09951a01-a5c0-4e24-9e51-70994d3b0bd0"}
{"level":"info","ts":"2026-09-18T09:58:01.227Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"09951a01-a5c0-4e24-9e51-70994d3b0bd0"}
{"level":"info","ts":"2026-09-18T09:58:01.227Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"09951a01-a5c0-4e24-9e51-70994d3b0bd0","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:01.227Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"09951a01-a5c0-4e24-9e51-70994d3b0bd0","error":"head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"error","ts":"2026-09-18T09:58:01.227Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"09951a01-a5c0-4e24-9e51-70994d3b0bd0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:01.227Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"09951a01-a5c0-4e24-9e51-70994d3b0bd0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"info","ts":"2026-09-18T09:58:01.227Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"09951a01-a5c0-4e24-9e51-70994d3b0bd0","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:01.227Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"09951a01-a5c0-4e24-9e51-70994d3b0bd0","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:01.227Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"09951a01-a5c0-4e24-9e51-70994d3b0bd0","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:01.227Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"09951a01-a5c0-4e24-9e51-70994d3b0bd0"}
{"level":"error","ts":"2026-09-18T09:58:01.227Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"09951a01-a5c0-4e24-9e51-70994d3b0bd0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:01.548Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"3f966941-cb67-4d94-8ab2-db4fff2caca3"}
{"level":"info","ts":"2026-09-18T09:58:01.548Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"3f966941-cb67-4d94-8ab2-db4fff2caca3"}
{"level":"info","ts":"2026-09-18T09:58:01.548Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"3f966941-cb67-4d94-8ab2-db4fff2caca3"}
{"level":"info","ts":"2026-09-18T09:58:01.548Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"3f966941-cb67-4d94-8ab2-db4fff2caca3","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:01.548Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"3f966941-cb67-4d94-8ab2-db4fff2caca3","error":"head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"error","ts":"2026-09-18T09:58:01.548Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"3f966941-cb67-4d94-8ab2-db4fff2caca3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:01.548Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"3f966941-cb67-4d94-8ab2-db4fff2caca3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"info","ts":"2026-09-18T09:58:01.549Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"3f966941-cb67-4d94-8ab2-db4fff2caca3","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:01.549Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"3f966941-cb67-4d94-8ab2-db4fff2caca3","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:01.549Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"3f966941-cb67-4d94-8ab2-db4fff2caca3","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:01.549Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"3f966941-cb67-4d94-8ab2-db4fff2caca3"}
{"level":"error","ts":"2026-09-18T09:58:01.549Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"3f966941-cb67-4d94-8ab2-db4fff2caca3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:01.690Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-ltx56"},"reconcileID":"110bd5de-afd8-4832-b873-d9749a32a92d"}
{"level":"info","ts":"2026-09-18T09:58:02.189Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"7f006fa9-66b5-4663-afd0-394ad7020743"}
{"level":"info","ts":"2026-09-18T09:58:02.189Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"7f006fa9-66b5-4663-afd0-394ad7020743"}
{"level":"info","ts":"2026-09-18T09:58:02.189Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"7f006fa9-66b5-4663-afd0-394ad7020743"}
{"level":"info","ts":"2026-09-18T09:58:02.189Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"7f006fa9-66b5-4663-afd0-394ad7020743","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:02.189Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"7f006fa9-66b5-4663-afd0-394ad7020743","error":"head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"error","ts":"2026-09-18T09:58:02.189Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"7f006fa9-66b5-4663-afd0-394ad7020743","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:02.189Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"7f006fa9-66b5-4663-afd0-394ad7020743","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found"}
{"level":"info","ts":"2026-09-18T09:58:02.189Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"7f006fa9-66b5-4663-afd0-394ad7020743","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:02.189Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"7f006fa9-66b5-4663-afd0-394ad7020743","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:02.189Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"7f006fa9-66b5-4663-afd0-394ad7020743","filter labels":["test-ns-ltx56",{"ray.io/cluster":"fail-k8s-job-q64ml","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T09:58:02.189Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"7f006fa9-66b5-4663-afd0-394ad7020743"}
{"level":"error","ts":"2026-09-18T09:58:02.189Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"7f006fa9-66b5-4663-afd0-394ad7020743","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-q64ml not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:03.470Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0"}
{"level":"info","ts":"2026-09-18T09:58:03.470Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0"}
{"level":"info","ts":"2026-09-18T09:58:03.470Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0"}
{"level":"info","ts":"2026-09-18T09:58:03.470Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:03.470Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0"}
{"level":"info","ts":"2026-09-18T09:58:03.470Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:58:03.470Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0"}
{"level":"info","ts":"2026-09-18T09:58:03.470Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","mode":"IntegratedOAuth","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:03.470Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","cluster":"fail-k8s-job-q64ml","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T09:58:03.470Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","cluster":"fail-k8s-job-q64ml","authType":"IntegratedOAuth","serviceAccount":"fail-k8s-job-q64ml-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T09:58:03.470Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-k8s-job-q64ml","ray.io/group":"headgroup","ray.io/identifier":"fail-k8s-job-q64ml-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T09:58:03.470Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T09:58:03.470Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:58:03.470Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T09:58:03.470Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","enabled":true}
{"level":"info","ts":"2026-09-18T09:58:03.489Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","name":"fail-k8s-job-q64ml-head-clvdq"}
{"level":"info","ts":"2026-09-18T09:58:03.489Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:58:03.489Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:03.489Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T09:58:03.489Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T09:58:03.489Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:03.489Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T09:58:03.489Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0"}
{"level":"info","ts":"2026-09-18T09:58:03.489Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","nodeType":"worker","rayStartParams":{"address":"fail-k8s-job-q64ml-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T09:58:03.489Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","rayStartCmd":"ray start  --address=fail-k8s-job-q64ml-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:58:03.489Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-k8s-job-q64ml-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T09:58:03.489Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","enabled":true}
{"level":"info","ts":"2026-09-18T09:58:03.503Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","name":"fail-k8s-job-q64ml-small-group-worker-9sjpx"}
{"level":"info","ts":"2026-09-18T09:58:03.503Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:03.503Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:03.503Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"5cb6781d-eda1-4158-bac0-a5fcc95eefa0","name":"fail-k8s-job-q64ml","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:58:03.517Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944"}
{"level":"info","ts":"2026-09-18T09:58:03.517Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944"}
{"level":"info","ts":"2026-09-18T09:58:03.518Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944"}
{"level":"info","ts":"2026-09-18T09:58:03.518Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:03.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:58:03.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:03.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:58:03.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:03.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:03.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:58:03.518Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:58:03.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:03.518Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:03.518Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"7b3bcaa2-abe9-4202-be54-7d001c606af0","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:03.518Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:03.566Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944","name":"fail-k8s-job-q64ml","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:58:03.666Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"64bdcac8-7f3e-4ed3-b2b6-ea173fa24fde","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:03.666Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"64bdcac8-7f3e-4ed3-b2b6-ea173fa24fde","rayCluster":"fail-k8s-job-q64ml","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:58:03.666Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"6ecee1e9-37e8-4ec5-94e7-da4dc09f7c36","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:03.666Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"6ecee1e9-37e8-4ec5-94e7-da4dc09f7c36","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:03.666Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"6ecee1e9-37e8-4ec5-94e7-da4dc09f7c36","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:03.667Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"6ecee1e9-37e8-4ec5-94e7-da4dc09f7c36"}
{"level":"info","ts":"2026-09-18T09:58:03.667Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"6ecee1e9-37e8-4ec5-94e7-da4dc09f7c36","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:03.667Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"6ecee1e9-37e8-4ec5-94e7-da4dc09f7c36","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:03.667Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"6ecee1e9-37e8-4ec5-94e7-da4dc09f7c36","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:58:03.675Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944","name":"fail-k8s-job-q64ml","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-q64ml\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml"}}
{"level":"info","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944"}
{"level":"error","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"958c13c9-8d75-4135-9120-a03686a46944","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-q64ml\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fbf3ca5f-b633-4629-8861-1b13f7832d17"}
{"level":"info","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fbf3ca5f-b633-4629-8861-1b13f7832d17"}
{"level":"info","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fbf3ca5f-b633-4629-8861-1b13f7832d17"}
{"level":"info","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fbf3ca5f-b633-4629-8861-1b13f7832d17","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fbf3ca5f-b633-4629-8861-1b13f7832d17","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fbf3ca5f-b633-4629-8861-1b13f7832d17","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fbf3ca5f-b633-4629-8861-1b13f7832d17","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fbf3ca5f-b633-4629-8861-1b13f7832d17","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fbf3ca5f-b633-4629-8861-1b13f7832d17","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fbf3ca5f-b633-4629-8861-1b13f7832d17","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fbf3ca5f-b633-4629-8861-1b13f7832d17","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fbf3ca5f-b633-4629-8861-1b13f7832d17","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fbf3ca5f-b633-4629-8861-1b13f7832d17","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fbf3ca5f-b633-4629-8861-1b13f7832d17","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:03.676Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fbf3ca5f-b633-4629-8861-1b13f7832d17","name":"fail-k8s-job-q64ml","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:58:03.684Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"7b3bcaa2-abe9-4202-be54-7d001c606af0","name":"fail-k8s-job-q64ml-head"}
{"level":"info","ts":"2026-09-18T09:58:03.686Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"03d8745a-100f-40c1-b7cb-05789862c5ec","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:03.686Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"03d8745a-100f-40c1-b7cb-05789862c5ec","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:03.686Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"03d8745a-100f-40c1-b7cb-05789862c5ec","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:03.686Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"03d8745a-100f-40c1-b7cb-05789862c5ec"}
{"level":"info","ts":"2026-09-18T09:58:03.686Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"03d8745a-100f-40c1-b7cb-05789862c5ec","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:03.686Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"03d8745a-100f-40c1-b7cb-05789862c5ec","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a"}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a"}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a"}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"28a69988-1a5b-4bc3-9900-e180705cbe70","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"28a69988-1a5b-4bc3-9900-e180705cbe70","rayCluster":"fail-k8s-job-q64ml","podIPs":[]}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a","name":"fail-k8s-job-q64ml","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"45ccf07b-a654-4a2c-b89d-0e5f5d27bb0c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"45ccf07b-a654-4a2c-b89d-0e5f5d27bb0c","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:03.690Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"45ccf07b-a654-4a2c-b89d-0e5f5d27bb0c","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:03.766Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"45ccf07b-a654-4a2c-b89d-0e5f5d27bb0c"}
{"level":"info","ts":"2026-09-18T09:58:03.766Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"45ccf07b-a654-4a2c-b89d-0e5f5d27bb0c","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:03.766Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"45ccf07b-a654-4a2c-b89d-0e5f5d27bb0c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:03.766Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"45ccf07b-a654-4a2c-b89d-0e5f5d27bb0c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:58:03.771Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"7b3bcaa2-abe9-4202-be54-7d001c606af0","name":"fail-k8s-job-q64ml-workers"}
{"level":"info","ts":"2026-09-18T09:58:03.771Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"7b3bcaa2-abe9-4202-be54-7d001c606af0","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:03.771Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"bff18cdc-94c9-4e54-97fa-a6bb8836ddb9","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:03.774Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a","name":"fail-k8s-job-q64ml","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-q64ml\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml"}}
{"level":"info","ts":"2026-09-18T09:58:03.774Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a"}
{"level":"error","ts":"2026-09-18T09:58:03.774Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ec1f4329-ba4e-48d8-866a-ca6d85724f7a","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-q64ml\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:03.779Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"949b7424-aa18-4400-8e8b-26f15f938735"}
{"level":"info","ts":"2026-09-18T09:58:03.779Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"949b7424-aa18-4400-8e8b-26f15f938735"}
{"level":"info","ts":"2026-09-18T09:58:03.779Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"949b7424-aa18-4400-8e8b-26f15f938735"}
{"level":"info","ts":"2026-09-18T09:58:03.780Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"949b7424-aa18-4400-8e8b-26f15f938735","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:03.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"949b7424-aa18-4400-8e8b-26f15f938735","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:58:03.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"949b7424-aa18-4400-8e8b-26f15f938735","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:03.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"949b7424-aa18-4400-8e8b-26f15f938735","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:58:03.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"949b7424-aa18-4400-8e8b-26f15f938735","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:03.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"949b7424-aa18-4400-8e8b-26f15f938735","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:03.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"949b7424-aa18-4400-8e8b-26f15f938735","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:58:03.780Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"949b7424-aa18-4400-8e8b-26f15f938735","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:58:03.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"949b7424-aa18-4400-8e8b-26f15f938735","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:03.780Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"949b7424-aa18-4400-8e8b-26f15f938735","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:03.780Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"949b7424-aa18-4400-8e8b-26f15f938735","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:03.780Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"949b7424-aa18-4400-8e8b-26f15f938735","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:58:03.780Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"949b7424-aa18-4400-8e8b-26f15f938735","seconds":300}
{"level":"info","ts":"2026-09-18T09:58:03.785Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"edc140e4-7c82-45bd-8556-8f702c781dcb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:03.785Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"edc140e4-7c82-45bd-8556-8f702c781dcb","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:03.785Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"edc140e4-7c82-45bd-8556-8f702c781dcb","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:03.786Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"edc140e4-7c82-45bd-8556-8f702c781dcb"}
{"level":"info","ts":"2026-09-18T09:58:03.786Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"edc140e4-7c82-45bd-8556-8f702c781dcb","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:03.786Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"edc140e4-7c82-45bd-8556-8f702c781dcb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:03.786Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"edc140e4-7c82-45bd-8556-8f702c781dcb","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:58:03.788Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"bff18cdc-94c9-4e54-97fa-a6bb8836ddb9","name":"fail-k8s-job-q64ml-head"}
{"level":"info","ts":"2026-09-18T09:58:03.794Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"edc140e4-7c82-45bd-8556-8f702c781dcb","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail-k8s-job\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T09:58:03.795Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"edc140e4-7c82-45bd-8556-8f702c781dcb"}
{"level":"error","ts":"2026-09-18T09:58:03.795Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"edc140e4-7c82-45bd-8556-8f702c781dcb","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail-k8s-job\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:58:03.795Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"be6291d5-0963-43c3-ad6a-32b82488f4b2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:03.795Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"be6291d5-0963-43c3-ad6a-32b82488f4b2","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:03.795Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"be6291d5-0963-43c3-ad6a-32b82488f4b2","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:03.795Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"be6291d5-0963-43c3-ad6a-32b82488f4b2"}
{"level":"info","ts":"2026-09-18T09:58:03.795Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"be6291d5-0963-43c3-ad6a-32b82488f4b2","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:03.795Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"be6291d5-0963-43c3-ad6a-32b82488f4b2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:03.800Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"661592a2-d521-4592-bb0b-ac8998eaf241","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:03.800Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"661592a2-d521-4592-bb0b-ac8998eaf241","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:03.800Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"661592a2-d521-4592-bb0b-ac8998eaf241","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:03.800Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"661592a2-d521-4592-bb0b-ac8998eaf241"}
{"level":"info","ts":"2026-09-18T09:58:03.800Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"661592a2-d521-4592-bb0b-ac8998eaf241","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:03.800Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"661592a2-d521-4592-bb0b-ac8998eaf241","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:03.801Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"bff18cdc-94c9-4e54-97fa-a6bb8836ddb9","name":"fail-k8s-job-q64ml-workers"}
{"level":"info","ts":"2026-09-18T09:58:03.801Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"bff18cdc-94c9-4e54-97fa-a6bb8836ddb9","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:04.087Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"bc23aad6-9388-4050-bede-43a33699a9f7"}
{"level":"info","ts":"2026-09-18T09:58:04.087Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"bc23aad6-9388-4050-bede-43a33699a9f7"}
{"level":"info","ts":"2026-09-18T09:58:04.087Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"bc23aad6-9388-4050-bede-43a33699a9f7"}
{"level":"info","ts":"2026-09-18T09:58:04.088Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"bc23aad6-9388-4050-bede-43a33699a9f7","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:04.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"bc23aad6-9388-4050-bede-43a33699a9f7","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:58:04.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"bc23aad6-9388-4050-bede-43a33699a9f7","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:04.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"bc23aad6-9388-4050-bede-43a33699a9f7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:58:04.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"bc23aad6-9388-4050-bede-43a33699a9f7","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:04.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"bc23aad6-9388-4050-bede-43a33699a9f7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:04.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"bc23aad6-9388-4050-bede-43a33699a9f7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:58:04.088Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"bc23aad6-9388-4050-bede-43a33699a9f7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:58:04.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"bc23aad6-9388-4050-bede-43a33699a9f7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:04.088Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"bc23aad6-9388-4050-bede-43a33699a9f7","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:04.088Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"bc23aad6-9388-4050-bede-43a33699a9f7","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:04.088Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"bc23aad6-9388-4050-bede-43a33699a9f7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:58:04.088Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"bc23aad6-9388-4050-bede-43a33699a9f7","seconds":300}
{"level":"info","ts":"2026-09-18T09:58:04.104Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fdfe09b7-099e-4667-bbf2-089d5375c25b"}
{"level":"info","ts":"2026-09-18T09:58:04.104Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fdfe09b7-099e-4667-bbf2-089d5375c25b"}
{"level":"info","ts":"2026-09-18T09:58:04.104Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fdfe09b7-099e-4667-bbf2-089d5375c25b"}
{"level":"info","ts":"2026-09-18T09:58:04.104Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fdfe09b7-099e-4667-bbf2-089d5375c25b","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:04.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fdfe09b7-099e-4667-bbf2-089d5375c25b","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:58:04.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fdfe09b7-099e-4667-bbf2-089d5375c25b","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:04.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fdfe09b7-099e-4667-bbf2-089d5375c25b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:58:04.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fdfe09b7-099e-4667-bbf2-089d5375c25b","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:04.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fdfe09b7-099e-4667-bbf2-089d5375c25b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:04.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fdfe09b7-099e-4667-bbf2-089d5375c25b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:58:04.105Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fdfe09b7-099e-4667-bbf2-089d5375c25b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:58:04.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fdfe09b7-099e-4667-bbf2-089d5375c25b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:04.105Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fdfe09b7-099e-4667-bbf2-089d5375c25b","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:04.105Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fdfe09b7-099e-4667-bbf2-089d5375c25b","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:04.105Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fdfe09b7-099e-4667-bbf2-089d5375c25b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:58:04.105Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fdfe09b7-099e-4667-bbf2-089d5375c25b","seconds":300}
{"level":"info","ts":"2026-09-18T09:58:04.309Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"75784537-70ea-4e48-93bd-d2956000180e"}
{"level":"info","ts":"2026-09-18T09:58:04.309Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"75784537-70ea-4e48-93bd-d2956000180e"}
{"level":"info","ts":"2026-09-18T09:58:04.309Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"75784537-70ea-4e48-93bd-d2956000180e"}
{"level":"info","ts":"2026-09-18T09:58:04.309Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"75784537-70ea-4e48-93bd-d2956000180e","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:04.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"75784537-70ea-4e48-93bd-d2956000180e","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:58:04.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"75784537-70ea-4e48-93bd-d2956000180e","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:04.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"75784537-70ea-4e48-93bd-d2956000180e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:58:04.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"75784537-70ea-4e48-93bd-d2956000180e","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:04.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"75784537-70ea-4e48-93bd-d2956000180e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:04.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"75784537-70ea-4e48-93bd-d2956000180e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:58:04.310Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"75784537-70ea-4e48-93bd-d2956000180e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:58:04.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"75784537-70ea-4e48-93bd-d2956000180e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:04.310Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"75784537-70ea-4e48-93bd-d2956000180e","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:04.310Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"75784537-70ea-4e48-93bd-d2956000180e","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:04.310Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"75784537-70ea-4e48-93bd-d2956000180e","name":"fail-k8s-job-q64ml","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:58:04.322Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"0384d6da-8c63-4b53-b617-b48d1a8ac1a5","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:04.322Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"e49d72ea-21df-4bed-8a7c-297c69b8d497","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:04.322Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"e49d72ea-21df-4bed-8a7c-297c69b8d497","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:04.322Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"e49d72ea-21df-4bed-8a7c-297c69b8d497","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:04.322Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"e49d72ea-21df-4bed-8a7c-297c69b8d497"}
{"level":"info","ts":"2026-09-18T09:58:04.322Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"e49d72ea-21df-4bed-8a7c-297c69b8d497","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:04.322Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"e49d72ea-21df-4bed-8a7c-297c69b8d497","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:04.322Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"e49d72ea-21df-4bed-8a7c-297c69b8d497","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:58:04.322Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"3ac97d5a-6ab4-4696-a6dc-436855a75f7e","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:04.322Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"3ac97d5a-6ab4-4696-a6dc-436855a75f7e","pod":"fail-k8s-job-q64ml-head-clvdq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-09-18T09:58:04.332Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"0384d6da-8c63-4b53-b617-b48d1a8ac1a5","name":"fail-k8s-job-q64ml-head"}
{"level":"info","ts":"2026-09-18T09:58:04.334Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a635e915-febe-448e-8688-c7d71a7de08e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:04.334Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a635e915-febe-448e-8688-c7d71a7de08e","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:04.334Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a635e915-febe-448e-8688-c7d71a7de08e","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:04.334Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a635e915-febe-448e-8688-c7d71a7de08e"}
{"level":"info","ts":"2026-09-18T09:58:04.334Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a635e915-febe-448e-8688-c7d71a7de08e","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:04.334Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a635e915-febe-448e-8688-c7d71a7de08e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:04.336Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"3ac97d5a-6ab4-4696-a6dc-436855a75f7e","certificate":"ray-head-cert-fail-k8s-job-q64ml","dnsNames":["fail-k8s-job-q64ml-head-svc","localhost","fail-k8s-job-q64ml-head-svc.test-ns-ltx56.svc","fail-k8s-job-q64ml-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.40","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:58:04.343Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"0384d6da-8c63-4b53-b617-b48d1a8ac1a5","name":"fail-k8s-job-q64ml-workers"}
{"level":"info","ts":"2026-09-18T09:58:04.343Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"0384d6da-8c63-4b53-b617-b48d1a8ac1a5","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:04.351Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"3ac97d5a-6ab4-4696-a6dc-436855a75f7e","certificate":"ray-worker-cert-fail-k8s-job-q64ml","dnsNames":["fail-k8s-job-q64ml-worker-svc","localhost","fail-k8s-job-q64ml-small-group","fail-k8s-job-q64ml-worker-svc.test-ns-ltx56.svc","fail-k8s-job-q64ml-worker-svc.test-ns-ltx56.svc.cluster.local","*.fail-k8s-job-q64ml-worker-svc.test-ns-ltx56.svc","*.fail-k8s-job-q64ml-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.40","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:58:04.351Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"3ac97d5a-6ab4-4696-a6dc-436855a75f7e","rayCluster":"fail-k8s-job-q64ml","podIPs":["10.133.0.40"]}
{"level":"info","ts":"2026-09-18T09:58:04.720Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"9d9d99a9-02cf-49e6-850c-750c502d6cff"}
{"level":"info","ts":"2026-09-18T09:58:04.720Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"9d9d99a9-02cf-49e6-850c-750c502d6cff"}
{"level":"info","ts":"2026-09-18T09:58:04.720Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"9d9d99a9-02cf-49e6-850c-750c502d6cff"}
{"level":"info","ts":"2026-09-18T09:58:04.720Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"9d9d99a9-02cf-49e6-850c-750c502d6cff","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:04.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"9d9d99a9-02cf-49e6-850c-750c502d6cff","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:58:04.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"9d9d99a9-02cf-49e6-850c-750c502d6cff","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:04.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"9d9d99a9-02cf-49e6-850c-750c502d6cff","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:58:04.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"9d9d99a9-02cf-49e6-850c-750c502d6cff","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:04.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"9d9d99a9-02cf-49e6-850c-750c502d6cff","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:04.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"9d9d99a9-02cf-49e6-850c-750c502d6cff","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:58:04.721Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"9d9d99a9-02cf-49e6-850c-750c502d6cff","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:58:04.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"9d9d99a9-02cf-49e6-850c-750c502d6cff","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:04.721Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"9d9d99a9-02cf-49e6-850c-750c502d6cff","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:04.721Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"9d9d99a9-02cf-49e6-850c-750c502d6cff","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:04.721Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"9d9d99a9-02cf-49e6-850c-750c502d6cff","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:58:04.721Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"9d9d99a9-02cf-49e6-850c-750c502d6cff","seconds":300}
{"level":"info","ts":"2026-09-18T09:58:06.323Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"db195ed8-bc76-4a26-89cc-127b5cbfa5d6"}
{"level":"info","ts":"2026-09-18T09:58:06.323Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"db195ed8-bc76-4a26-89cc-127b5cbfa5d6"}
{"level":"info","ts":"2026-09-18T09:58:06.323Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"db195ed8-bc76-4a26-89cc-127b5cbfa5d6"}
{"level":"info","ts":"2026-09-18T09:58:06.323Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"db195ed8-bc76-4a26-89cc-127b5cbfa5d6","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:06.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"db195ed8-bc76-4a26-89cc-127b5cbfa5d6","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:58:06.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"db195ed8-bc76-4a26-89cc-127b5cbfa5d6","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:06.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"db195ed8-bc76-4a26-89cc-127b5cbfa5d6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:58:06.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"db195ed8-bc76-4a26-89cc-127b5cbfa5d6","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:06.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"db195ed8-bc76-4a26-89cc-127b5cbfa5d6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:06.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"db195ed8-bc76-4a26-89cc-127b5cbfa5d6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:58:06.323Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"db195ed8-bc76-4a26-89cc-127b5cbfa5d6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:58:06.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"db195ed8-bc76-4a26-89cc-127b5cbfa5d6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:06.323Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"db195ed8-bc76-4a26-89cc-127b5cbfa5d6","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:06.323Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"db195ed8-bc76-4a26-89cc-127b5cbfa5d6","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:06.323Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"db195ed8-bc76-4a26-89cc-127b5cbfa5d6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:58:06.323Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"db195ed8-bc76-4a26-89cc-127b5cbfa5d6","seconds":300}
{"level":"info","ts":"2026-09-18T09:58:06.801Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f643ba7e-2a7b-4a43-ae23-771203f82c3a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:06.801Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f643ba7e-2a7b-4a43-ae23-771203f82c3a","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:06.801Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f643ba7e-2a7b-4a43-ae23-771203f82c3a","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:06.801Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f643ba7e-2a7b-4a43-ae23-771203f82c3a"}
{"level":"info","ts":"2026-09-18T09:58:06.801Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f643ba7e-2a7b-4a43-ae23-771203f82c3a","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:06.801Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f643ba7e-2a7b-4a43-ae23-771203f82c3a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:09.332Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6bfe65fd-cbee-46f1-9b20-6b03dea4f7c2"}
{"level":"info","ts":"2026-09-18T09:58:09.332Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6bfe65fd-cbee-46f1-9b20-6b03dea4f7c2"}
{"level":"info","ts":"2026-09-18T09:58:09.332Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6bfe65fd-cbee-46f1-9b20-6b03dea4f7c2"}
{"level":"info","ts":"2026-09-18T09:58:09.332Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6bfe65fd-cbee-46f1-9b20-6b03dea4f7c2","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:09.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6bfe65fd-cbee-46f1-9b20-6b03dea4f7c2","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:58:09.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6bfe65fd-cbee-46f1-9b20-6b03dea4f7c2","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:09.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6bfe65fd-cbee-46f1-9b20-6b03dea4f7c2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:58:09.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6bfe65fd-cbee-46f1-9b20-6b03dea4f7c2","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:09.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6bfe65fd-cbee-46f1-9b20-6b03dea4f7c2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:09.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6bfe65fd-cbee-46f1-9b20-6b03dea4f7c2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:58:09.333Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6bfe65fd-cbee-46f1-9b20-6b03dea4f7c2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:58:09.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6bfe65fd-cbee-46f1-9b20-6b03dea4f7c2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:09.333Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6bfe65fd-cbee-46f1-9b20-6b03dea4f7c2","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:09.333Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6bfe65fd-cbee-46f1-9b20-6b03dea4f7c2","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:09.333Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6bfe65fd-cbee-46f1-9b20-6b03dea4f7c2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:58:09.333Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"6bfe65fd-cbee-46f1-9b20-6b03dea4f7c2","seconds":300}
{"level":"info","ts":"2026-09-18T09:58:09.802Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d1e3a4b6-3d6f-463d-be1a-a73733eae9dd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:09.802Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d1e3a4b6-3d6f-463d-be1a-a73733eae9dd","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:09.802Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d1e3a4b6-3d6f-463d-be1a-a73733eae9dd","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:09.802Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d1e3a4b6-3d6f-463d-be1a-a73733eae9dd"}
{"level":"info","ts":"2026-09-18T09:58:09.802Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d1e3a4b6-3d6f-463d-be1a-a73733eae9dd","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:09.802Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d1e3a4b6-3d6f-463d-be1a-a73733eae9dd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:10.342Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"00965765-8884-498b-851f-d662da5800b9"}
{"level":"info","ts":"2026-09-18T09:58:10.342Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"00965765-8884-498b-851f-d662da5800b9"}
{"level":"info","ts":"2026-09-18T09:58:10.342Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"00965765-8884-498b-851f-d662da5800b9"}
{"level":"info","ts":"2026-09-18T09:58:10.342Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"00965765-8884-498b-851f-d662da5800b9","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:10.342Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"00965765-8884-498b-851f-d662da5800b9","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:58:10.342Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"00965765-8884-498b-851f-d662da5800b9","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:10.342Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"00965765-8884-498b-851f-d662da5800b9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:58:10.342Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"00965765-8884-498b-851f-d662da5800b9","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:10.342Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"00965765-8884-498b-851f-d662da5800b9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:10.342Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"00965765-8884-498b-851f-d662da5800b9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:58:10.342Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"00965765-8884-498b-851f-d662da5800b9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:58:10.342Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"00965765-8884-498b-851f-d662da5800b9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:10.342Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"00965765-8884-498b-851f-d662da5800b9","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:58:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:58:10.342Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"00965765-8884-498b-851f-d662da5800b9","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:58:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:58:10.342Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"00965765-8884-498b-851f-d662da5800b9","name":"fail-k8s-job-q64ml","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:58:10.354Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"54e0831c-3b02-4951-8632-2ed1715d1cb9","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:10.355Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"da60620b-8e80-4988-8e7c-c16bd8b9fd71","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:10.355Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"da60620b-8e80-4988-8e7c-c16bd8b9fd71","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:10.355Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"da60620b-8e80-4988-8e7c-c16bd8b9fd71","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:10.355Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"da60620b-8e80-4988-8e7c-c16bd8b9fd71"}
{"level":"info","ts":"2026-09-18T09:58:10.355Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"da60620b-8e80-4988-8e7c-c16bd8b9fd71","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:10.355Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"da60620b-8e80-4988-8e7c-c16bd8b9fd71","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:10.355Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"da60620b-8e80-4988-8e7c-c16bd8b9fd71","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:58:10.355Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"cadd93d7-144c-42c0-a8d0-84712b713507","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:10.355Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"cadd93d7-144c-42c0-a8d0-84712b713507","pod":"fail-k8s-job-q64ml-head-clvdq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-09-18T09:58:10.355Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"cadd93d7-144c-42c0-a8d0-84712b713507","pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","ip":"10.132.0.41"}
{"level":"info","ts":"2026-09-18T09:58:10.365Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"54e0831c-3b02-4951-8632-2ed1715d1cb9","name":"fail-k8s-job-q64ml-head"}
{"level":"info","ts":"2026-09-18T09:58:10.367Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d3f9f922-8eb8-4866-9537-e4b7d6bb0f03","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:10.367Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d3f9f922-8eb8-4866-9537-e4b7d6bb0f03","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:10.367Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d3f9f922-8eb8-4866-9537-e4b7d6bb0f03","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:10.368Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d3f9f922-8eb8-4866-9537-e4b7d6bb0f03"}
{"level":"info","ts":"2026-09-18T09:58:10.368Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d3f9f922-8eb8-4866-9537-e4b7d6bb0f03","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:10.368Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d3f9f922-8eb8-4866-9537-e4b7d6bb0f03","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:10.369Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"cadd93d7-144c-42c0-a8d0-84712b713507","certificate":"ray-head-cert-fail-k8s-job-q64ml","dnsNames":["fail-k8s-job-q64ml-head-svc","localhost","fail-k8s-job-q64ml-head-svc.test-ns-ltx56.svc","fail-k8s-job-q64ml-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.40","10.132.0.41","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:58:10.376Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"54e0831c-3b02-4951-8632-2ed1715d1cb9","name":"fail-k8s-job-q64ml-workers"}
{"level":"info","ts":"2026-09-18T09:58:10.376Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"54e0831c-3b02-4951-8632-2ed1715d1cb9","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:10.386Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"cadd93d7-144c-42c0-a8d0-84712b713507","certificate":"ray-worker-cert-fail-k8s-job-q64ml","dnsNames":["fail-k8s-job-q64ml-worker-svc","localhost","fail-k8s-job-q64ml-small-group","fail-k8s-job-q64ml-worker-svc.test-ns-ltx56.svc","fail-k8s-job-q64ml-worker-svc.test-ns-ltx56.svc.cluster.local","*.fail-k8s-job-q64ml-worker-svc.test-ns-ltx56.svc","*.fail-k8s-job-q64ml-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.40","10.132.0.41","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T09:58:10.386Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"cadd93d7-144c-42c0-a8d0-84712b713507"}
{"level":"info","ts":"2026-09-18T09:58:12.355Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ae88069e-5db3-4775-aa2c-3c8c2d97fe5d"}
{"level":"info","ts":"2026-09-18T09:58:12.355Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ae88069e-5db3-4775-aa2c-3c8c2d97fe5d"}
{"level":"info","ts":"2026-09-18T09:58:12.355Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ae88069e-5db3-4775-aa2c-3c8c2d97fe5d"}
{"level":"info","ts":"2026-09-18T09:58:12.355Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ae88069e-5db3-4775-aa2c-3c8c2d97fe5d","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:12.355Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ae88069e-5db3-4775-aa2c-3c8c2d97fe5d","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:58:12.355Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ae88069e-5db3-4775-aa2c-3c8c2d97fe5d","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:12.355Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ae88069e-5db3-4775-aa2c-3c8c2d97fe5d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:58:12.355Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ae88069e-5db3-4775-aa2c-3c8c2d97fe5d","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:12.355Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ae88069e-5db3-4775-aa2c-3c8c2d97fe5d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:12.355Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ae88069e-5db3-4775-aa2c-3c8c2d97fe5d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:58:12.355Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ae88069e-5db3-4775-aa2c-3c8c2d97fe5d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:58:12.355Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ae88069e-5db3-4775-aa2c-3c8c2d97fe5d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:12.355Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ae88069e-5db3-4775-aa2c-3c8c2d97fe5d","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:58:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:58:12.355Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ae88069e-5db3-4775-aa2c-3c8c2d97fe5d","podName":"fail-k8s-job-q64ml-head-clvdq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:58:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T09:58:12.355Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ae88069e-5db3-4775-aa2c-3c8c2d97fe5d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:58:12.355Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"ae88069e-5db3-4775-aa2c-3c8c2d97fe5d","seconds":300}
{"level":"info","ts":"2026-09-18T09:58:12.803Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5eba1c74-98ae-46eb-a6ca-6ec0ac9ab42a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:12.803Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5eba1c74-98ae-46eb-a6ca-6ec0ac9ab42a","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:12.803Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5eba1c74-98ae-46eb-a6ca-6ec0ac9ab42a","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:12.803Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5eba1c74-98ae-46eb-a6ca-6ec0ac9ab42a"}
{"level":"info","ts":"2026-09-18T09:58:12.803Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5eba1c74-98ae-46eb-a6ca-6ec0ac9ab42a","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:12.803Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5eba1c74-98ae-46eb-a6ca-6ec0ac9ab42a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:15.804Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"98c98102-2de8-4d18-9b29-fb34c660f6ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:15.804Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"98c98102-2de8-4d18-9b29-fb34c660f6ff","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:15.804Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"98c98102-2de8-4d18-9b29-fb34c660f6ff","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:15.804Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"98c98102-2de8-4d18-9b29-fb34c660f6ff"}
{"level":"info","ts":"2026-09-18T09:58:15.804Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"98c98102-2de8-4d18-9b29-fb34c660f6ff","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:15.804Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"98c98102-2de8-4d18-9b29-fb34c660f6ff","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:18.804Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"acd09229-cae3-496b-b717-af0f3344e978","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:18.804Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"acd09229-cae3-496b-b717-af0f3344e978","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:18.804Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"acd09229-cae3-496b-b717-af0f3344e978","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:18.804Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"acd09229-cae3-496b-b717-af0f3344e978"}
{"level":"info","ts":"2026-09-18T09:58:18.804Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"acd09229-cae3-496b-b717-af0f3344e978","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:18.804Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"acd09229-cae3-496b-b717-af0f3344e978","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:21.374Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fcedaf0b-3a62-44fa-83e4-c2d0ad8c9e12"}
{"level":"info","ts":"2026-09-18T09:58:21.374Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fcedaf0b-3a62-44fa-83e4-c2d0ad8c9e12"}
{"level":"info","ts":"2026-09-18T09:58:21.374Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fcedaf0b-3a62-44fa-83e4-c2d0ad8c9e12"}
{"level":"info","ts":"2026-09-18T09:58:21.374Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fcedaf0b-3a62-44fa-83e4-c2d0ad8c9e12","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:21.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fcedaf0b-3a62-44fa-83e4-c2d0ad8c9e12","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:58:21.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fcedaf0b-3a62-44fa-83e4-c2d0ad8c9e12","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:21.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fcedaf0b-3a62-44fa-83e4-c2d0ad8c9e12","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:58:21.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fcedaf0b-3a62-44fa-83e4-c2d0ad8c9e12","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:21.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fcedaf0b-3a62-44fa-83e4-c2d0ad8c9e12","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:21.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fcedaf0b-3a62-44fa-83e4-c2d0ad8c9e12","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:58:21.375Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fcedaf0b-3a62-44fa-83e4-c2d0ad8c9e12","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:58:21.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fcedaf0b-3a62-44fa-83e4-c2d0ad8c9e12","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:21.375Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fcedaf0b-3a62-44fa-83e4-c2d0ad8c9e12","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:21.375Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fcedaf0b-3a62-44fa-83e4-c2d0ad8c9e12","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:21.375Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"fcedaf0b-3a62-44fa-83e4-c2d0ad8c9e12","name":"fail-k8s-job-q64ml","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:58:21.390Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c3fafd95-6ba1-4154-a51a-f76c30b4e598","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:21.390Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c3fafd95-6ba1-4154-a51a-f76c30b4e598","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:21.390Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c3fafd95-6ba1-4154-a51a-f76c30b4e598","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:21.391Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"d59c8f8b-ab42-4003-919e-7bf4f9e34163","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:21.391Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c3fafd95-6ba1-4154-a51a-f76c30b4e598"}
{"level":"info","ts":"2026-09-18T09:58:21.391Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c3fafd95-6ba1-4154-a51a-f76c30b4e598","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:21.391Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c3fafd95-6ba1-4154-a51a-f76c30b4e598","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:21.391Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c3fafd95-6ba1-4154-a51a-f76c30b4e598","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:58:21.391Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"458d270f-2b43-42f6-afc0-7601fe6beea3","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:21.391Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"458d270f-2b43-42f6-afc0-7601fe6beea3","pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","ip":"10.132.0.41"}
{"level":"info","ts":"2026-09-18T09:58:21.391Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"458d270f-2b43-42f6-afc0-7601fe6beea3","pod":"fail-k8s-job-q64ml-head-clvdq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-09-18T09:58:21.391Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"458d270f-2b43-42f6-afc0-7601fe6beea3","rayCluster":"fail-k8s-job-q64ml","podIPs":["10.132.0.41","10.133.0.40"]}
{"level":"info","ts":"2026-09-18T09:58:21.416Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b56f8a42-33dd-48f1-a5d1-023e8bc94660","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:21.416Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b56f8a42-33dd-48f1-a5d1-023e8bc94660","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:21.416Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b56f8a42-33dd-48f1-a5d1-023e8bc94660","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:21.416Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b56f8a42-33dd-48f1-a5d1-023e8bc94660"}
{"level":"info","ts":"2026-09-18T09:58:21.416Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b56f8a42-33dd-48f1-a5d1-023e8bc94660","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:21.416Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b56f8a42-33dd-48f1-a5d1-023e8bc94660","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:21.421Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"d59c8f8b-ab42-4003-919e-7bf4f9e34163","name":"fail-k8s-job-q64ml-head"}
{"level":"info","ts":"2026-09-18T09:58:21.437Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"d59c8f8b-ab42-4003-919e-7bf4f9e34163","name":"fail-k8s-job-q64ml-workers"}
{"level":"info","ts":"2026-09-18T09:58:21.437Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"d59c8f8b-ab42-4003-919e-7bf4f9e34163","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:21.805Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1bb475ef-8ca2-4a8f-b5cd-08bd68534458","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:21.805Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1bb475ef-8ca2-4a8f-b5cd-08bd68534458","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:21.805Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1bb475ef-8ca2-4a8f-b5cd-08bd68534458","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:21.806Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1bb475ef-8ca2-4a8f-b5cd-08bd68534458"}
{"level":"info","ts":"2026-09-18T09:58:21.806Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1bb475ef-8ca2-4a8f-b5cd-08bd68534458","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:21.806Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1bb475ef-8ca2-4a8f-b5cd-08bd68534458","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:23.391Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8a628b35-e735-45e7-b0aa-9fcf87201359"}
{"level":"info","ts":"2026-09-18T09:58:23.391Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8a628b35-e735-45e7-b0aa-9fcf87201359"}
{"level":"info","ts":"2026-09-18T09:58:23.391Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8a628b35-e735-45e7-b0aa-9fcf87201359"}
{"level":"info","ts":"2026-09-18T09:58:23.391Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8a628b35-e735-45e7-b0aa-9fcf87201359","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:58:23.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8a628b35-e735-45e7-b0aa-9fcf87201359","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:58:23.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8a628b35-e735-45e7-b0aa-9fcf87201359","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:23.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8a628b35-e735-45e7-b0aa-9fcf87201359","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:58:23.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8a628b35-e735-45e7-b0aa-9fcf87201359","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:58:23.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8a628b35-e735-45e7-b0aa-9fcf87201359","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:23.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8a628b35-e735-45e7-b0aa-9fcf87201359","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:58:23.391Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8a628b35-e735-45e7-b0aa-9fcf87201359","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:58:23.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8a628b35-e735-45e7-b0aa-9fcf87201359","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:58:23.391Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8a628b35-e735-45e7-b0aa-9fcf87201359","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:23.391Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8a628b35-e735-45e7-b0aa-9fcf87201359","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:58:23.391Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8a628b35-e735-45e7-b0aa-9fcf87201359","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:58:23.391Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8a628b35-e735-45e7-b0aa-9fcf87201359","seconds":300}
{"level":"info","ts":"2026-09-18T09:58:24.806Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"fbf9372d-8449-464e-9e0b-62e12940a849","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:24.806Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"fbf9372d-8449-464e-9e0b-62e12940a849","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:24.806Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"fbf9372d-8449-464e-9e0b-62e12940a849","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:24.806Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"fbf9372d-8449-464e-9e0b-62e12940a849"}
{"level":"info","ts":"2026-09-18T09:58:24.806Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"fbf9372d-8449-464e-9e0b-62e12940a849","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:24.806Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"fbf9372d-8449-464e-9e0b-62e12940a849","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:27.807Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f57b5276-0a37-40bd-8969-606f8c2f765e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:27.807Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f57b5276-0a37-40bd-8969-606f8c2f765e","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:27.807Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f57b5276-0a37-40bd-8969-606f8c2f765e","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:27.807Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f57b5276-0a37-40bd-8969-606f8c2f765e"}
{"level":"info","ts":"2026-09-18T09:58:27.807Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f57b5276-0a37-40bd-8969-606f8c2f765e","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:27.807Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f57b5276-0a37-40bd-8969-606f8c2f765e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:30.808Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"2b787201-4008-42b5-960c-5c3f6371036e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:30.808Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"2b787201-4008-42b5-960c-5c3f6371036e","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:30.808Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"2b787201-4008-42b5-960c-5c3f6371036e","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:30.808Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"2b787201-4008-42b5-960c-5c3f6371036e"}
{"level":"info","ts":"2026-09-18T09:58:30.808Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"2b787201-4008-42b5-960c-5c3f6371036e","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:30.808Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"2b787201-4008-42b5-960c-5c3f6371036e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:31.067Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"db48644d-402d-4a73-8709-66da7960557b","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:31.067Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"db48644d-402d-4a73-8709-66da7960557b","pod":"fail-k8s-job-q64ml-head-clvdq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-09-18T09:58:31.067Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"db48644d-402d-4a73-8709-66da7960557b","pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","ip":"10.132.0.41"}
{"level":"info","ts":"2026-09-18T09:58:31.067Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"db48644d-402d-4a73-8709-66da7960557b","rayCluster":"fail-k8s-job-q64ml","podIPs":["10.133.0.40","10.132.0.41"]}
{"level":"info","ts":"2026-09-18T09:58:33.809Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d9ef1cd8-1af7-41e1-9153-3a11494f76c0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:33.809Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d9ef1cd8-1af7-41e1-9153-3a11494f76c0","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:33.809Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d9ef1cd8-1af7-41e1-9153-3a11494f76c0","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:33.809Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d9ef1cd8-1af7-41e1-9153-3a11494f76c0"}
{"level":"info","ts":"2026-09-18T09:58:33.809Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d9ef1cd8-1af7-41e1-9153-3a11494f76c0","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:33.809Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d9ef1cd8-1af7-41e1-9153-3a11494f76c0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:36.809Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d4d5a882-23ee-423a-8fff-79468bcfc0a0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:36.809Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d4d5a882-23ee-423a-8fff-79468bcfc0a0","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:36.809Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d4d5a882-23ee-423a-8fff-79468bcfc0a0","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:36.809Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d4d5a882-23ee-423a-8fff-79468bcfc0a0"}
{"level":"info","ts":"2026-09-18T09:58:36.809Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d4d5a882-23ee-423a-8fff-79468bcfc0a0","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:36.809Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d4d5a882-23ee-423a-8fff-79468bcfc0a0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:39.810Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"dbb53f87-0b55-4d9b-b667-80b81f24d421","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:39.810Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"dbb53f87-0b55-4d9b-b667-80b81f24d421","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:39.810Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"dbb53f87-0b55-4d9b-b667-80b81f24d421","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:39.810Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"dbb53f87-0b55-4d9b-b667-80b81f24d421"}
{"level":"info","ts":"2026-09-18T09:58:39.810Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"dbb53f87-0b55-4d9b-b667-80b81f24d421","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:39.810Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"dbb53f87-0b55-4d9b-b667-80b81f24d421","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:42.811Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"2d42f9fa-94e6-4012-81ae-7a9dc70d2c68","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:42.812Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"2d42f9fa-94e6-4012-81ae-7a9dc70d2c68","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:42.812Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"2d42f9fa-94e6-4012-81ae-7a9dc70d2c68","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:42.812Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"2d42f9fa-94e6-4012-81ae-7a9dc70d2c68"}
{"level":"info","ts":"2026-09-18T09:58:42.812Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"2d42f9fa-94e6-4012-81ae-7a9dc70d2c68","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:42.812Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"2d42f9fa-94e6-4012-81ae-7a9dc70d2c68","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:45.813Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"28bb5374-8112-4691-9493-d3a51fed2454","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:45.813Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"28bb5374-8112-4691-9493-d3a51fed2454","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:45.813Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"28bb5374-8112-4691-9493-d3a51fed2454","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:45.813Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"28bb5374-8112-4691-9493-d3a51fed2454"}
{"level":"info","ts":"2026-09-18T09:58:45.813Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"28bb5374-8112-4691-9493-d3a51fed2454","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:45.813Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"28bb5374-8112-4691-9493-d3a51fed2454","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:48.814Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"48398f85-bd6d-49d0-ac93-f0b0358ef2e1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:48.814Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"48398f85-bd6d-49d0-ac93-f0b0358ef2e1","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:48.814Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"48398f85-bd6d-49d0-ac93-f0b0358ef2e1","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:48.814Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"48398f85-bd6d-49d0-ac93-f0b0358ef2e1"}
{"level":"info","ts":"2026-09-18T09:58:48.814Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"48398f85-bd6d-49d0-ac93-f0b0358ef2e1","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:48.814Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"48398f85-bd6d-49d0-ac93-f0b0358ef2e1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:51.815Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5ea16ff5-02b6-420e-a4f7-bd8ea1dd9cf4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:51.815Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5ea16ff5-02b6-420e-a4f7-bd8ea1dd9cf4","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:51.815Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5ea16ff5-02b6-420e-a4f7-bd8ea1dd9cf4","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:51.815Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5ea16ff5-02b6-420e-a4f7-bd8ea1dd9cf4"}
{"level":"info","ts":"2026-09-18T09:58:51.815Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5ea16ff5-02b6-420e-a4f7-bd8ea1dd9cf4","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:51.815Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5ea16ff5-02b6-420e-a4f7-bd8ea1dd9cf4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:54.816Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"9baed270-6cbc-46f2-bf89-e7faf46257af","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:54.816Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"9baed270-6cbc-46f2-bf89-e7faf46257af","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:54.816Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"9baed270-6cbc-46f2-bf89-e7faf46257af","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:54.816Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"9baed270-6cbc-46f2-bf89-e7faf46257af"}
{"level":"info","ts":"2026-09-18T09:58:54.816Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"9baed270-6cbc-46f2-bf89-e7faf46257af","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:54.816Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"9baed270-6cbc-46f2-bf89-e7faf46257af","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:58:57.817Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"98b26e1f-c5ae-440a-8fb8-c05a4384da24","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:58:57.817Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"98b26e1f-c5ae-440a-8fb8-c05a4384da24","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:58:57.817Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"98b26e1f-c5ae-440a-8fb8-c05a4384da24","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:58:57.817Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"98b26e1f-c5ae-440a-8fb8-c05a4384da24"}
{"level":"info","ts":"2026-09-18T09:58:57.817Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"98b26e1f-c5ae-440a-8fb8-c05a4384da24","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:58:57.817Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"98b26e1f-c5ae-440a-8fb8-c05a4384da24","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:00.818Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b9433e23-7fb9-47f6-ad6f-fc9883dc525c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:00.818Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b9433e23-7fb9-47f6-ad6f-fc9883dc525c","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:00.818Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b9433e23-7fb9-47f6-ad6f-fc9883dc525c","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:00.818Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b9433e23-7fb9-47f6-ad6f-fc9883dc525c"}
{"level":"info","ts":"2026-09-18T09:59:00.818Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b9433e23-7fb9-47f6-ad6f-fc9883dc525c","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:00.818Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b9433e23-7fb9-47f6-ad6f-fc9883dc525c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:03.819Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"7dd1f3e2-8bf1-4e3a-8478-5f1d8240a96f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:03.819Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"7dd1f3e2-8bf1-4e3a-8478-5f1d8240a96f","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:03.819Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"7dd1f3e2-8bf1-4e3a-8478-5f1d8240a96f","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:03.819Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"7dd1f3e2-8bf1-4e3a-8478-5f1d8240a96f"}
{"level":"info","ts":"2026-09-18T09:59:03.819Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"7dd1f3e2-8bf1-4e3a-8478-5f1d8240a96f","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:03.819Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"7dd1f3e2-8bf1-4e3a-8478-5f1d8240a96f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:06.820Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"40e30659-b06d-4436-9f22-f14b9b8f01f4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:06.820Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"40e30659-b06d-4436-9f22-f14b9b8f01f4","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:06.820Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"40e30659-b06d-4436-9f22-f14b9b8f01f4","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:06.820Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"40e30659-b06d-4436-9f22-f14b9b8f01f4"}
{"level":"info","ts":"2026-09-18T09:59:06.820Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"40e30659-b06d-4436-9f22-f14b9b8f01f4","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:06.820Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"40e30659-b06d-4436-9f22-f14b9b8f01f4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:09.821Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"55722491-7db0-453e-b79b-38d8ba84616d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:09.821Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"55722491-7db0-453e-b79b-38d8ba84616d","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:09.821Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"55722491-7db0-453e-b79b-38d8ba84616d","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:09.821Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"55722491-7db0-453e-b79b-38d8ba84616d"}
{"level":"info","ts":"2026-09-18T09:59:09.821Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"55722491-7db0-453e-b79b-38d8ba84616d","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:09.821Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"55722491-7db0-453e-b79b-38d8ba84616d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:12.822Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b14b47f1-bc5c-47ee-922e-298c5c0832ac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:12.822Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b14b47f1-bc5c-47ee-922e-298c5c0832ac","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:12.822Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b14b47f1-bc5c-47ee-922e-298c5c0832ac","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:12.822Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b14b47f1-bc5c-47ee-922e-298c5c0832ac"}
{"level":"info","ts":"2026-09-18T09:59:12.822Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b14b47f1-bc5c-47ee-922e-298c5c0832ac","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:12.822Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b14b47f1-bc5c-47ee-922e-298c5c0832ac","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:15.823Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d4217af1-289e-422e-a7a6-91dfd697c181","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:15.823Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d4217af1-289e-422e-a7a6-91dfd697c181","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:15.823Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d4217af1-289e-422e-a7a6-91dfd697c181","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:15.823Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d4217af1-289e-422e-a7a6-91dfd697c181"}
{"level":"info","ts":"2026-09-18T09:59:15.823Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d4217af1-289e-422e-a7a6-91dfd697c181","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:15.823Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d4217af1-289e-422e-a7a6-91dfd697c181","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:18.824Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"dbc0e2cf-153c-4cbb-85f4-2894d2ef3fc9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:18.824Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"dbc0e2cf-153c-4cbb-85f4-2894d2ef3fc9","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:18.824Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"dbc0e2cf-153c-4cbb-85f4-2894d2ef3fc9","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:18.824Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"dbc0e2cf-153c-4cbb-85f4-2894d2ef3fc9"}
{"level":"info","ts":"2026-09-18T09:59:18.824Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"dbc0e2cf-153c-4cbb-85f4-2894d2ef3fc9","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:18.824Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"dbc0e2cf-153c-4cbb-85f4-2894d2ef3fc9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:21.825Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"fa6e29e4-7d2b-4c37-ae8c-8c2f461841eb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:21.825Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"fa6e29e4-7d2b-4c37-ae8c-8c2f461841eb","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:21.825Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"fa6e29e4-7d2b-4c37-ae8c-8c2f461841eb","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:21.825Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"fa6e29e4-7d2b-4c37-ae8c-8c2f461841eb"}
{"level":"info","ts":"2026-09-18T09:59:21.825Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"fa6e29e4-7d2b-4c37-ae8c-8c2f461841eb","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:21.825Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"fa6e29e4-7d2b-4c37-ae8c-8c2f461841eb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:24.826Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8bb807d1-6c60-40d3-a6b8-f4498970b750","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:24.826Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8bb807d1-6c60-40d3-a6b8-f4498970b750","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:24.826Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8bb807d1-6c60-40d3-a6b8-f4498970b750","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:24.827Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8bb807d1-6c60-40d3-a6b8-f4498970b750"}
{"level":"info","ts":"2026-09-18T09:59:24.827Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8bb807d1-6c60-40d3-a6b8-f4498970b750","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:24.827Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8bb807d1-6c60-40d3-a6b8-f4498970b750","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:27.827Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"bff80fb2-2139-49df-99e2-d26091830457","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:27.827Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"bff80fb2-2139-49df-99e2-d26091830457","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:27.827Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"bff80fb2-2139-49df-99e2-d26091830457","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:27.827Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"bff80fb2-2139-49df-99e2-d26091830457"}
{"level":"info","ts":"2026-09-18T09:59:27.827Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"bff80fb2-2139-49df-99e2-d26091830457","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:27.827Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"bff80fb2-2139-49df-99e2-d26091830457","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:30.828Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1227f7ae-d9c7-4b2d-9c3c-717004122ee8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:30.828Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1227f7ae-d9c7-4b2d-9c3c-717004122ee8","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:30.828Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1227f7ae-d9c7-4b2d-9c3c-717004122ee8","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:30.828Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1227f7ae-d9c7-4b2d-9c3c-717004122ee8"}
{"level":"info","ts":"2026-09-18T09:59:30.828Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1227f7ae-d9c7-4b2d-9c3c-717004122ee8","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:30.828Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1227f7ae-d9c7-4b2d-9c3c-717004122ee8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:31.068Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"271aa1ea-3cd2-4341-8b3b-c1f6e5d542f5","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:31.068Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"271aa1ea-3cd2-4341-8b3b-c1f6e5d542f5","pod":"fail-k8s-job-q64ml-head-clvdq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-09-18T09:59:31.068Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"271aa1ea-3cd2-4341-8b3b-c1f6e5d542f5","pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","ip":"10.132.0.41"}
{"level":"info","ts":"2026-09-18T09:59:31.068Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"271aa1ea-3cd2-4341-8b3b-c1f6e5d542f5","rayCluster":"fail-k8s-job-q64ml","podIPs":["10.133.0.40","10.132.0.41"]}
{"level":"info","ts":"2026-09-18T09:59:33.829Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8c45e744-ebe0-4dc8-89f0-fa8324cbe937","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:33.829Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8c45e744-ebe0-4dc8-89f0-fa8324cbe937","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:33.829Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8c45e744-ebe0-4dc8-89f0-fa8324cbe937","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:33.829Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8c45e744-ebe0-4dc8-89f0-fa8324cbe937"}
{"level":"info","ts":"2026-09-18T09:59:33.829Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8c45e744-ebe0-4dc8-89f0-fa8324cbe937","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:33.829Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8c45e744-ebe0-4dc8-89f0-fa8324cbe937","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:35.150Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"411db9f7-0544-4eab-81ce-9664fd10585b"}
{"level":"info","ts":"2026-09-18T09:59:35.150Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"411db9f7-0544-4eab-81ce-9664fd10585b"}
{"level":"info","ts":"2026-09-18T09:59:35.150Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"411db9f7-0544-4eab-81ce-9664fd10585b"}
{"level":"info","ts":"2026-09-18T09:59:35.150Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"411db9f7-0544-4eab-81ce-9664fd10585b","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:59:35.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"411db9f7-0544-4eab-81ce-9664fd10585b","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:59:35.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"411db9f7-0544-4eab-81ce-9664fd10585b","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:59:35.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"411db9f7-0544-4eab-81ce-9664fd10585b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:59:35.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"411db9f7-0544-4eab-81ce-9664fd10585b","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:59:35.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"411db9f7-0544-4eab-81ce-9664fd10585b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:59:35.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"411db9f7-0544-4eab-81ce-9664fd10585b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:59:35.151Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"411db9f7-0544-4eab-81ce-9664fd10585b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:59:35.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"411db9f7-0544-4eab-81ce-9664fd10585b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:59:35.151Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"411db9f7-0544-4eab-81ce-9664fd10585b","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:59:35.151Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"411db9f7-0544-4eab-81ce-9664fd10585b","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:59:35.151Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"411db9f7-0544-4eab-81ce-9664fd10585b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:59:35.151Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"411db9f7-0544-4eab-81ce-9664fd10585b","seconds":300}
{"level":"info","ts":"2026-09-18T09:59:36.152Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"885d56f6-2f25-40c2-994e-a17ac7b4e5cb"}
{"level":"info","ts":"2026-09-18T09:59:36.152Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"885d56f6-2f25-40c2-994e-a17ac7b4e5cb"}
{"level":"info","ts":"2026-09-18T09:59:36.152Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"885d56f6-2f25-40c2-994e-a17ac7b4e5cb"}
{"level":"info","ts":"2026-09-18T09:59:36.152Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"885d56f6-2f25-40c2-994e-a17ac7b4e5cb","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:59:36.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"885d56f6-2f25-40c2-994e-a17ac7b4e5cb","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:59:36.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"885d56f6-2f25-40c2-994e-a17ac7b4e5cb","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:59:36.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"885d56f6-2f25-40c2-994e-a17ac7b4e5cb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:59:36.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"885d56f6-2f25-40c2-994e-a17ac7b4e5cb","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:59:36.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"885d56f6-2f25-40c2-994e-a17ac7b4e5cb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:59:36.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"885d56f6-2f25-40c2-994e-a17ac7b4e5cb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:59:36.152Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"885d56f6-2f25-40c2-994e-a17ac7b4e5cb","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:59:36.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"885d56f6-2f25-40c2-994e-a17ac7b4e5cb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:59:36.153Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"885d56f6-2f25-40c2-994e-a17ac7b4e5cb","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:59:36.153Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"885d56f6-2f25-40c2-994e-a17ac7b4e5cb","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:59:36.153Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"885d56f6-2f25-40c2-994e-a17ac7b4e5cb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:59:36.153Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"885d56f6-2f25-40c2-994e-a17ac7b4e5cb","seconds":300}
{"level":"info","ts":"2026-09-18T09:59:36.830Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"767870c0-a467-47d8-b2ac-529d99893814","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:36.830Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"767870c0-a467-47d8-b2ac-529d99893814","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:36.830Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"767870c0-a467-47d8-b2ac-529d99893814","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:36.830Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"767870c0-a467-47d8-b2ac-529d99893814"}
{"level":"info","ts":"2026-09-18T09:59:36.830Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"767870c0-a467-47d8-b2ac-529d99893814","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:36.830Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"767870c0-a467-47d8-b2ac-529d99893814","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:39.830Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"47ffec0a-06f2-4448-beb6-386811323ec4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:39.830Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"47ffec0a-06f2-4448-beb6-386811323ec4","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:39.830Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"47ffec0a-06f2-4448-beb6-386811323ec4","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:39.830Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"47ffec0a-06f2-4448-beb6-386811323ec4"}
{"level":"info","ts":"2026-09-18T09:59:39.830Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"47ffec0a-06f2-4448-beb6-386811323ec4","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:39.830Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"47ffec0a-06f2-4448-beb6-386811323ec4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:42.831Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8bc80661-fa18-407e-bd15-614ae88c8ba5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:42.831Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8bc80661-fa18-407e-bd15-614ae88c8ba5","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:42.831Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8bc80661-fa18-407e-bd15-614ae88c8ba5","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:42.832Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8bc80661-fa18-407e-bd15-614ae88c8ba5"}
{"level":"info","ts":"2026-09-18T09:59:42.832Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8bc80661-fa18-407e-bd15-614ae88c8ba5","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:42.832Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8bc80661-fa18-407e-bd15-614ae88c8ba5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:45.198Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"b0889a63-98c0-4018-bdd0-d68c5c4fb46c"}
{"level":"info","ts":"2026-09-18T09:59:45.198Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"b0889a63-98c0-4018-bdd0-d68c5c4fb46c"}
{"level":"info","ts":"2026-09-18T09:59:45.198Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"b0889a63-98c0-4018-bdd0-d68c5c4fb46c"}
{"level":"info","ts":"2026-09-18T09:59:45.198Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"b0889a63-98c0-4018-bdd0-d68c5c4fb46c","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:59:45.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"b0889a63-98c0-4018-bdd0-d68c5c4fb46c","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:59:45.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"b0889a63-98c0-4018-bdd0-d68c5c4fb46c","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:59:45.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"b0889a63-98c0-4018-bdd0-d68c5c4fb46c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:59:45.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"b0889a63-98c0-4018-bdd0-d68c5c4fb46c","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:59:45.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"b0889a63-98c0-4018-bdd0-d68c5c4fb46c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:59:45.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"b0889a63-98c0-4018-bdd0-d68c5c4fb46c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:59:45.199Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"b0889a63-98c0-4018-bdd0-d68c5c4fb46c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:59:45.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"b0889a63-98c0-4018-bdd0-d68c5c4fb46c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:59:45.199Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"b0889a63-98c0-4018-bdd0-d68c5c4fb46c","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:59:45.199Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"b0889a63-98c0-4018-bdd0-d68c5c4fb46c","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T09:59:45.199Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"b0889a63-98c0-4018-bdd0-d68c5c4fb46c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:59:45.199Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"b0889a63-98c0-4018-bdd0-d68c5c4fb46c","seconds":300}
{"level":"info","ts":"2026-09-18T09:59:45.832Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c397611b-f113-45d0-ae63-1926318cfdb0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:45.832Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c397611b-f113-45d0-ae63-1926318cfdb0","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:45.832Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c397611b-f113-45d0-ae63-1926318cfdb0","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:45.832Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c397611b-f113-45d0-ae63-1926318cfdb0"}
{"level":"info","ts":"2026-09-18T09:59:45.832Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c397611b-f113-45d0-ae63-1926318cfdb0","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:45.832Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c397611b-f113-45d0-ae63-1926318cfdb0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:46.212Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a094221a-ae64-4c28-a707-094edbab9b8b"}
{"level":"info","ts":"2026-09-18T09:59:46.212Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a094221a-ae64-4c28-a707-094edbab9b8b"}
{"level":"info","ts":"2026-09-18T09:59:46.212Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a094221a-ae64-4c28-a707-094edbab9b8b"}
{"level":"info","ts":"2026-09-18T09:59:46.212Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a094221a-ae64-4c28-a707-094edbab9b8b","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:59:46.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a094221a-ae64-4c28-a707-094edbab9b8b","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:59:46.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a094221a-ae64-4c28-a707-094edbab9b8b","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:59:46.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a094221a-ae64-4c28-a707-094edbab9b8b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:59:46.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a094221a-ae64-4c28-a707-094edbab9b8b","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:59:46.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a094221a-ae64-4c28-a707-094edbab9b8b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:59:46.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a094221a-ae64-4c28-a707-094edbab9b8b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:59:46.212Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a094221a-ae64-4c28-a707-094edbab9b8b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:59:46.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a094221a-ae64-4c28-a707-094edbab9b8b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:59:46.212Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a094221a-ae64-4c28-a707-094edbab9b8b","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:58:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:59:46.212Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a094221a-ae64-4c28-a707-094edbab9b8b","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:58:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:59:46.212Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"a094221a-ae64-4c28-a707-094edbab9b8b","name":"fail-k8s-job-q64ml","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:59:46.230Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"f84f4ece-5a3b-403b-b223-dc390d9d19f9","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:46.230Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"42a4e94e-1bcd-4c29-a7d1-e5f17804d837","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:46.230Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"42a4e94e-1bcd-4c29-a7d1-e5f17804d837","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:46.230Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"42a4e94e-1bcd-4c29-a7d1-e5f17804d837","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:46.230Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"42a4e94e-1bcd-4c29-a7d1-e5f17804d837"}
{"level":"info","ts":"2026-09-18T09:59:46.230Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"42a4e94e-1bcd-4c29-a7d1-e5f17804d837","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:46.230Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"42a4e94e-1bcd-4c29-a7d1-e5f17804d837","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:58:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:46.230Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"42a4e94e-1bcd-4c29-a7d1-e5f17804d837","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T09:59:46.230Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"641cb20f-7eaa-43ba-ba2c-ef9f78dd0fb0","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:46.230Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"641cb20f-7eaa-43ba-ba2c-ef9f78dd0fb0","pod":"fail-k8s-job-q64ml-head-clvdq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-09-18T09:59:46.230Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"641cb20f-7eaa-43ba-ba2c-ef9f78dd0fb0","pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","ip":"10.132.0.41"}
{"level":"info","ts":"2026-09-18T09:59:46.230Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"641cb20f-7eaa-43ba-ba2c-ef9f78dd0fb0","rayCluster":"fail-k8s-job-q64ml","podIPs":["10.133.0.40","10.132.0.41"]}
{"level":"info","ts":"2026-09-18T09:59:46.248Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"3f96c70c-507b-4765-810c-340a1c2e525f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:46.248Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"3f96c70c-507b-4765-810c-340a1c2e525f","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:46.248Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"3f96c70c-507b-4765-810c-340a1c2e525f","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:46.248Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"3f96c70c-507b-4765-810c-340a1c2e525f"}
{"level":"info","ts":"2026-09-18T09:59:46.248Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"3f96c70c-507b-4765-810c-340a1c2e525f","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:46.248Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"3f96c70c-507b-4765-810c-340a1c2e525f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:46.253Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"f84f4ece-5a3b-403b-b223-dc390d9d19f9","name":"fail-k8s-job-q64ml-head"}
{"level":"info","ts":"2026-09-18T09:59:46.265Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"f84f4ece-5a3b-403b-b223-dc390d9d19f9","name":"fail-k8s-job-q64ml-workers"}
{"level":"info","ts":"2026-09-18T09:59:46.265Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"f84f4ece-5a3b-403b-b223-dc390d9d19f9","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:48.230Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8d44621a-284f-46dd-8e18-6622d8af8eb6"}
{"level":"info","ts":"2026-09-18T09:59:48.230Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8d44621a-284f-46dd-8e18-6622d8af8eb6"}
{"level":"info","ts":"2026-09-18T09:59:48.230Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8d44621a-284f-46dd-8e18-6622d8af8eb6"}
{"level":"info","ts":"2026-09-18T09:59:48.230Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8d44621a-284f-46dd-8e18-6622d8af8eb6","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:59:48.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8d44621a-284f-46dd-8e18-6622d8af8eb6","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:59:48.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8d44621a-284f-46dd-8e18-6622d8af8eb6","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:59:48.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8d44621a-284f-46dd-8e18-6622d8af8eb6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:59:48.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8d44621a-284f-46dd-8e18-6622d8af8eb6","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:59:48.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8d44621a-284f-46dd-8e18-6622d8af8eb6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:59:48.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8d44621a-284f-46dd-8e18-6622d8af8eb6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:59:48.230Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8d44621a-284f-46dd-8e18-6622d8af8eb6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:59:48.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8d44621a-284f-46dd-8e18-6622d8af8eb6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:59:48.230Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8d44621a-284f-46dd-8e18-6622d8af8eb6","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:58:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:59:48.230Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8d44621a-284f-46dd-8e18-6622d8af8eb6","podName":"fail-k8s-job-q64ml-small-group-worker-9sjpx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T09:58:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T09:59:48.231Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8d44621a-284f-46dd-8e18-6622d8af8eb6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:59:48.231Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"8d44621a-284f-46dd-8e18-6622d8af8eb6","seconds":300}
{"level":"info","ts":"2026-09-18T09:59:48.833Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c6088a34-ff0e-44a1-a735-26faf60eff5c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:48.833Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c6088a34-ff0e-44a1-a735-26faf60eff5c","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:48.833Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c6088a34-ff0e-44a1-a735-26faf60eff5c","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:48.834Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c6088a34-ff0e-44a1-a735-26faf60eff5c"}
{"level":"info","ts":"2026-09-18T09:59:48.834Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c6088a34-ff0e-44a1-a735-26faf60eff5c","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:48.834Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c6088a34-ff0e-44a1-a735-26faf60eff5c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:51.835Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1fc7887b-7201-417a-a109-bdaa25a5560a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:51.835Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1fc7887b-7201-417a-a109-bdaa25a5560a","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:51.835Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1fc7887b-7201-417a-a109-bdaa25a5560a","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:51.835Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1fc7887b-7201-417a-a109-bdaa25a5560a"}
{"level":"info","ts":"2026-09-18T09:59:51.835Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1fc7887b-7201-417a-a109-bdaa25a5560a","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:51.835Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1fc7887b-7201-417a-a109-bdaa25a5560a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:54.836Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1babcc0f-3416-4056-be61-e07fb0b0000e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:54.836Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1babcc0f-3416-4056-be61-e07fb0b0000e","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:54.836Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1babcc0f-3416-4056-be61-e07fb0b0000e","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:54.836Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1babcc0f-3416-4056-be61-e07fb0b0000e"}
{"level":"info","ts":"2026-09-18T09:59:54.836Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1babcc0f-3416-4056-be61-e07fb0b0000e","RayCluster":"fail-k8s-job-q64ml","State":""}
{"level":"info","ts":"2026-09-18T09:59:54.836Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1babcc0f-3416-4056-be61-e07fb0b0000e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:57.223Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"714c49f0-aa9d-4177-ad4f-6bb14f053f8b"}
{"level":"info","ts":"2026-09-18T09:59:57.223Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"714c49f0-aa9d-4177-ad4f-6bb14f053f8b"}
{"level":"info","ts":"2026-09-18T09:59:57.223Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"714c49f0-aa9d-4177-ad4f-6bb14f053f8b"}
{"level":"info","ts":"2026-09-18T09:59:57.223Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"714c49f0-aa9d-4177-ad4f-6bb14f053f8b","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:59:57.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"714c49f0-aa9d-4177-ad4f-6bb14f053f8b","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:59:57.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"714c49f0-aa9d-4177-ad4f-6bb14f053f8b","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:59:57.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"714c49f0-aa9d-4177-ad4f-6bb14f053f8b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:59:57.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"714c49f0-aa9d-4177-ad4f-6bb14f053f8b","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:59:57.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"714c49f0-aa9d-4177-ad4f-6bb14f053f8b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:59:57.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"714c49f0-aa9d-4177-ad4f-6bb14f053f8b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:59:57.224Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"714c49f0-aa9d-4177-ad4f-6bb14f053f8b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:59:57.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"714c49f0-aa9d-4177-ad4f-6bb14f053f8b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:59:57.224Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"714c49f0-aa9d-4177-ad4f-6bb14f053f8b","name":"fail-k8s-job-q64ml","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:57Z","stateTransitionTimes":{"ready":"2026-09-18T09:59:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T09:59:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T09:59:57.235Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"eafa38df-88dc-4dda-97bd-1b242154cc91","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:57.235Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"22357674-478c-42a1-a2f5-002c5e631fae","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:57.235Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"22357674-478c-42a1-a2f5-002c5e631fae","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:57.235Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"22357674-478c-42a1-a2f5-002c5e631fae","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:57.236Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"22357674-478c-42a1-a2f5-002c5e631fae"}
{"level":"info","ts":"2026-09-18T09:59:57.236Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"323b8430-1ba9-40a1-a56f-4a2daf7c2e62","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:57.236Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"323b8430-1ba9-40a1-a56f-4a2daf7c2e62","pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","ip":"10.132.0.41"}
{"level":"info","ts":"2026-09-18T09:59:57.236Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"323b8430-1ba9-40a1-a56f-4a2daf7c2e62","pod":"fail-k8s-job-q64ml-head-clvdq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-09-18T09:59:57.236Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"323b8430-1ba9-40a1-a56f-4a2daf7c2e62","rayCluster":"fail-k8s-job-q64ml","podIPs":["10.132.0.41","10.133.0.40"]}
{"level":"info","ts":"2026-09-18T09:59:57.244Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"22357674-478c-42a1-a2f5-002c5e631fae","Kubernetes Job":"fail-k8s-job"}
{"level":"info","ts":"2026-09-18T09:59:57.244Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"22357674-478c-42a1-a2f5-002c5e631fae","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","dashboardURL":"fail-k8s-job-q64ml-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:57.244Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"22357674-478c-42a1-a2f5-002c5e631fae","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:59:57.248Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"eafa38df-88dc-4dda-97bd-1b242154cc91","name":"fail-k8s-job-q64ml-head"}
{"level":"info","ts":"2026-09-18T09:59:57.258Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"eafa38df-88dc-4dda-97bd-1b242154cc91","name":"fail-k8s-job-q64ml-workers"}
{"level":"info","ts":"2026-09-18T09:59:57.258Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"eafa38df-88dc-4dda-97bd-1b242154cc91","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:57.280Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"be594df4-5776-4e56-b3a8-546726c64dd5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:57.280Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"be594df4-5776-4e56-b3a8-546726c64dd5","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:57.280Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"be594df4-5776-4e56-b3a8-546726c64dd5","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:57.280Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"be594df4-5776-4e56-b3a8-546726c64dd5"}
{"level":"info","ts":"2026-09-18T09:59:57.280Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"be594df4-5776-4e56-b3a8-546726c64dd5","Kubernetes Job":"fail-k8s-job"}
{"level":"info","ts":"2026-09-18T09:59:57.280Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"be594df4-5776-4e56-b3a8-546726c64dd5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","dashboardURL":"fail-k8s-job-q64ml-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T09:59:57.280Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"be594df4-5776-4e56-b3a8-546726c64dd5","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T09:59:57.288Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"be594df4-5776-4e56-b3a8-546726c64dd5","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail-k8s-job\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T09:59:57.288Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"be594df4-5776-4e56-b3a8-546726c64dd5"}
{"level":"error","ts":"2026-09-18T09:59:57.288Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"be594df4-5776-4e56-b3a8-546726c64dd5","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail-k8s-job\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:57.288Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"9f28b6a1-4734-43fd-8fcd-b9459627e144","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:57.288Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"9f28b6a1-4734-43fd-8fcd-b9459627e144","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:57.288Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"9f28b6a1-4734-43fd-8fcd-b9459627e144","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:57.288Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"9f28b6a1-4734-43fd-8fcd-b9459627e144"}
{"level":"info","ts":"2026-09-18T09:59:57.288Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"9f28b6a1-4734-43fd-8fcd-b9459627e144","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:59:57.305Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"9f28b6a1-4734-43fd-8fcd-b9459627e144","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:57.305Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"9f28b6a1-4734-43fd-8fcd-b9459627e144"}
{"level":"error","ts":"2026-09-18T09:59:57.305Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"9f28b6a1-4734-43fd-8fcd-b9459627e144","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:57.305Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a7c099ba-726f-4cb7-9d7c-11ac58c5d7bd","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:57.305Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a7c099ba-726f-4cb7-9d7c-11ac58c5d7bd","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:57.305Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a7c099ba-726f-4cb7-9d7c-11ac58c5d7bd","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:57.305Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a7c099ba-726f-4cb7-9d7c-11ac58c5d7bd"}
{"level":"info","ts":"2026-09-18T09:59:57.305Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a7c099ba-726f-4cb7-9d7c-11ac58c5d7bd","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:59:57.309Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a7c099ba-726f-4cb7-9d7c-11ac58c5d7bd","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:57.309Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a7c099ba-726f-4cb7-9d7c-11ac58c5d7bd"}
{"level":"error","ts":"2026-09-18T09:59:57.309Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a7c099ba-726f-4cb7-9d7c-11ac58c5d7bd","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:57.316Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"eea9ea07-74e3-448d-bf76-88a269f85d7a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:57.316Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"eea9ea07-74e3-448d-bf76-88a269f85d7a","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:57.316Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"eea9ea07-74e3-448d-bf76-88a269f85d7a","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:57.316Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"eea9ea07-74e3-448d-bf76-88a269f85d7a"}
{"level":"info","ts":"2026-09-18T09:59:57.316Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"eea9ea07-74e3-448d-bf76-88a269f85d7a","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:59:57.320Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"eea9ea07-74e3-448d-bf76-88a269f85d7a","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:57.320Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"eea9ea07-74e3-448d-bf76-88a269f85d7a"}
{"level":"error","ts":"2026-09-18T09:59:57.320Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"eea9ea07-74e3-448d-bf76-88a269f85d7a","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:57.360Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"cc23071c-c03f-479a-805a-999717b70c2e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:57.360Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"cc23071c-c03f-479a-805a-999717b70c2e","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:57.360Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"cc23071c-c03f-479a-805a-999717b70c2e","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:57.361Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"cc23071c-c03f-479a-805a-999717b70c2e"}
{"level":"info","ts":"2026-09-18T09:59:57.361Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"cc23071c-c03f-479a-805a-999717b70c2e","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:59:57.365Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"cc23071c-c03f-479a-805a-999717b70c2e","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:57.365Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"cc23071c-c03f-479a-805a-999717b70c2e"}
{"level":"error","ts":"2026-09-18T09:59:57.365Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"cc23071c-c03f-479a-805a-999717b70c2e","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:57.445Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"bd2b3676-dcfb-4c62-a897-f975c0a8980f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:57.445Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"bd2b3676-dcfb-4c62-a897-f975c0a8980f","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:57.445Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"bd2b3676-dcfb-4c62-a897-f975c0a8980f","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:57.446Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"bd2b3676-dcfb-4c62-a897-f975c0a8980f"}
{"level":"info","ts":"2026-09-18T09:59:57.446Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"bd2b3676-dcfb-4c62-a897-f975c0a8980f","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:59:57.450Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"bd2b3676-dcfb-4c62-a897-f975c0a8980f","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:57.450Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"bd2b3676-dcfb-4c62-a897-f975c0a8980f"}
{"level":"error","ts":"2026-09-18T09:59:57.450Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"bd2b3676-dcfb-4c62-a897-f975c0a8980f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:57.611Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"eaa153ef-bc74-47bc-bc9c-3c98ad3b78d7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:57.611Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"eaa153ef-bc74-47bc-bc9c-3c98ad3b78d7","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:57.611Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"eaa153ef-bc74-47bc-bc9c-3c98ad3b78d7","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:57.611Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"eaa153ef-bc74-47bc-bc9c-3c98ad3b78d7"}
{"level":"info","ts":"2026-09-18T09:59:57.611Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"eaa153ef-bc74-47bc-bc9c-3c98ad3b78d7","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:59:57.616Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"eaa153ef-bc74-47bc-bc9c-3c98ad3b78d7","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:57.616Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"eaa153ef-bc74-47bc-bc9c-3c98ad3b78d7"}
{"level":"error","ts":"2026-09-18T09:59:57.616Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"eaa153ef-bc74-47bc-bc9c-3c98ad3b78d7","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:57.937Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b0e73765-a22b-4539-91b3-00ba70fe6dbb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:57.937Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b0e73765-a22b-4539-91b3-00ba70fe6dbb","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:57.937Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b0e73765-a22b-4539-91b3-00ba70fe6dbb","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:57.937Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b0e73765-a22b-4539-91b3-00ba70fe6dbb"}
{"level":"info","ts":"2026-09-18T09:59:57.937Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b0e73765-a22b-4539-91b3-00ba70fe6dbb","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:59:57.942Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b0e73765-a22b-4539-91b3-00ba70fe6dbb","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:57.942Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b0e73765-a22b-4539-91b3-00ba70fe6dbb"}
{"level":"error","ts":"2026-09-18T09:59:57.942Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b0e73765-a22b-4539-91b3-00ba70fe6dbb","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:58.268Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b9908139-d3a9-477f-92f4-92af29dd5611","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:58.268Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b9908139-d3a9-477f-92f4-92af29dd5611","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:58.268Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b9908139-d3a9-477f-92f4-92af29dd5611","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:58.269Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b9908139-d3a9-477f-92f4-92af29dd5611"}
{"level":"info","ts":"2026-09-18T09:59:58.269Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b9908139-d3a9-477f-92f4-92af29dd5611","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:59:58.369Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b9908139-d3a9-477f-92f4-92af29dd5611","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:58.369Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b9908139-d3a9-477f-92f4-92af29dd5611"}
{"level":"error","ts":"2026-09-18T09:59:58.369Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b9908139-d3a9-477f-92f4-92af29dd5611","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:58.583Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"4b1128a2-8bac-49b3-9614-186fbce1cb9c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T09:59:58.583Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"4b1128a2-8bac-49b3-9614-186fbce1cb9c","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T09:59:58.583Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"4b1128a2-8bac-49b3-9614-186fbce1cb9c","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T09:59:58.583Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"4b1128a2-8bac-49b3-9614-186fbce1cb9c"}
{"level":"info","ts":"2026-09-18T09:59:58.583Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"4b1128a2-8bac-49b3-9614-186fbce1cb9c","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T09:59:58.587Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"4b1128a2-8bac-49b3-9614-186fbce1cb9c","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:58.587Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"4b1128a2-8bac-49b3-9614-186fbce1cb9c"}
{"level":"error","ts":"2026-09-18T09:59:58.587Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"4b1128a2-8bac-49b3-9614-186fbce1cb9c","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T09:59:59.236Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c5994e01-5dee-491e-bd1e-90851b83fcc3"}
{"level":"info","ts":"2026-09-18T09:59:59.236Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c5994e01-5dee-491e-bd1e-90851b83fcc3"}
{"level":"info","ts":"2026-09-18T09:59:59.236Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c5994e01-5dee-491e-bd1e-90851b83fcc3"}
{"level":"info","ts":"2026-09-18T09:59:59.236Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c5994e01-5dee-491e-bd1e-90851b83fcc3","1 head service found":"fail-k8s-job-q64ml-head-svc"}
{"level":"info","ts":"2026-09-18T09:59:59.236Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c5994e01-5dee-491e-bd1e-90851b83fcc3","Found 1 head Pod":"fail-k8s-job-q64ml-head-clvdq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T09:59:59.236Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c5994e01-5dee-491e-bd1e-90851b83fcc3","head Pod":"fail-k8s-job-q64ml-head-clvdq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-q64ml-head-clvdq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:59:59.236Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c5994e01-5dee-491e-bd1e-90851b83fcc3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T09:59:59.236Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c5994e01-5dee-491e-bd1e-90851b83fcc3","worker Pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-q64ml-small-group-worker-9sjpx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T09:59:59.236Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c5994e01-5dee-491e-bd1e-90851b83fcc3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T09:59:59.236Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c5994e01-5dee-491e-bd1e-90851b83fcc3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T09:59:59.236Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c5994e01-5dee-491e-bd1e-90851b83fcc3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T09:59:59.236Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c5994e01-5dee-491e-bd1e-90851b83fcc3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T09:59:59.237Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c5994e01-5dee-491e-bd1e-90851b83fcc3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T09:59:59.237Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"c5994e01-5dee-491e-bd1e-90851b83fcc3","seconds":300}
{"level":"info","ts":"2026-09-18T10:00:01.148Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1b6219f8-3571-4c27-9320-8c237276eceb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:01.148Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1b6219f8-3571-4c27-9320-8c237276eceb","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:01.148Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1b6219f8-3571-4c27-9320-8c237276eceb","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:01.148Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1b6219f8-3571-4c27-9320-8c237276eceb"}
{"level":"info","ts":"2026-09-18T10:00:01.148Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1b6219f8-3571-4c27-9320-8c237276eceb","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:00:01.153Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1b6219f8-3571-4c27-9320-8c237276eceb","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:01.153Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1b6219f8-3571-4c27-9320-8c237276eceb"}
{"level":"error","ts":"2026-09-18T10:00:01.153Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"1b6219f8-3571-4c27-9320-8c237276eceb","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:03.284Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"0cdb84b5-dddb-4afb-820a-daa7ddac6978","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:03.284Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"0cdb84b5-dddb-4afb-820a-daa7ddac6978","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:03.284Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"0cdb84b5-dddb-4afb-820a-daa7ddac6978","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:03.284Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"0cdb84b5-dddb-4afb-820a-daa7ddac6978"}
{"level":"info","ts":"2026-09-18T10:00:03.285Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"0cdb84b5-dddb-4afb-820a-daa7ddac6978","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:00:03.289Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"0cdb84b5-dddb-4afb-820a-daa7ddac6978","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:03.289Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"0cdb84b5-dddb-4afb-820a-daa7ddac6978"}
{"level":"error","ts":"2026-09-18T10:00:03.289Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"0cdb84b5-dddb-4afb-820a-daa7ddac6978","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:04.292Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"14d2a9f2-c933-4656-bcfd-b5a2fc1cf3d6","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:04.292Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"14d2a9f2-c933-4656-bcfd-b5a2fc1cf3d6","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:04.292Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"14d2a9f2-c933-4656-bcfd-b5a2fc1cf3d6","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:04.292Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"14d2a9f2-c933-4656-bcfd-b5a2fc1cf3d6"}
{"level":"info","ts":"2026-09-18T10:00:04.292Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"14d2a9f2-c933-4656-bcfd-b5a2fc1cf3d6","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:00:04.297Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"14d2a9f2-c933-4656-bcfd-b5a2fc1cf3d6","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:04.297Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"14d2a9f2-c933-4656-bcfd-b5a2fc1cf3d6"}
{"level":"error","ts":"2026-09-18T10:00:04.297Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"14d2a9f2-c933-4656-bcfd-b5a2fc1cf3d6","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:04.309Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c03e58d6-e6e9-4b6b-8f49-99c0fce0786b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:04.309Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c03e58d6-e6e9-4b6b-8f49-99c0fce0786b","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:04.309Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c03e58d6-e6e9-4b6b-8f49-99c0fce0786b","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:04.309Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c03e58d6-e6e9-4b6b-8f49-99c0fce0786b"}
{"level":"info","ts":"2026-09-18T10:00:04.309Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c03e58d6-e6e9-4b6b-8f49-99c0fce0786b","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:00:04.313Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c03e58d6-e6e9-4b6b-8f49-99c0fce0786b","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:04.313Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c03e58d6-e6e9-4b6b-8f49-99c0fce0786b"}
{"level":"error","ts":"2026-09-18T10:00:04.313Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c03e58d6-e6e9-4b6b-8f49-99c0fce0786b","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:06.274Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5a5357ef-03b8-4038-be38-ef4d0bad0560","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:06.274Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5a5357ef-03b8-4038-be38-ef4d0bad0560","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:06.274Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5a5357ef-03b8-4038-be38-ef4d0bad0560","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:06.274Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5a5357ef-03b8-4038-be38-ef4d0bad0560"}
{"level":"info","ts":"2026-09-18T10:00:06.274Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5a5357ef-03b8-4038-be38-ef4d0bad0560","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:00:06.279Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5a5357ef-03b8-4038-be38-ef4d0bad0560","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:06.279Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5a5357ef-03b8-4038-be38-ef4d0bad0560"}
{"level":"error","ts":"2026-09-18T10:00:06.279Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"5a5357ef-03b8-4038-be38-ef4d0bad0560","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:12.026Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8b563edb-8e5a-476a-8361-7e088b976a03","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:12.026Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8b563edb-8e5a-476a-8361-7e088b976a03","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:12.027Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8b563edb-8e5a-476a-8361-7e088b976a03","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:12.027Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8b563edb-8e5a-476a-8361-7e088b976a03"}
{"level":"info","ts":"2026-09-18T10:00:12.027Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8b563edb-8e5a-476a-8361-7e088b976a03","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:00:12.032Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8b563edb-8e5a-476a-8361-7e088b976a03","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:12.032Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8b563edb-8e5a-476a-8361-7e088b976a03"}
{"level":"error","ts":"2026-09-18T10:00:12.032Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"8b563edb-8e5a-476a-8361-7e088b976a03","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:14.334Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"64d8cfab-9e2b-4d1d-b833-f4c0d30eb53e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:14.334Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"64d8cfab-9e2b-4d1d-b833-f4c0d30eb53e","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:14.335Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"64d8cfab-9e2b-4d1d-b833-f4c0d30eb53e","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:14.335Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"64d8cfab-9e2b-4d1d-b833-f4c0d30eb53e"}
{"level":"info","ts":"2026-09-18T10:00:14.335Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"64d8cfab-9e2b-4d1d-b833-f4c0d30eb53e","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:00:14.340Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"64d8cfab-9e2b-4d1d-b833-f4c0d30eb53e","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:14.340Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"64d8cfab-9e2b-4d1d-b833-f4c0d30eb53e"}
{"level":"error","ts":"2026-09-18T10:00:14.340Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"64d8cfab-9e2b-4d1d-b833-f4c0d30eb53e","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:18.355Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c260206d-0b8d-42e0-a178-e3377691c6f4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:18.355Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c260206d-0b8d-42e0-a178-e3377691c6f4","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:18.355Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c260206d-0b8d-42e0-a178-e3377691c6f4","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:18.355Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c260206d-0b8d-42e0-a178-e3377691c6f4"}
{"level":"info","ts":"2026-09-18T10:00:18.355Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c260206d-0b8d-42e0-a178-e3377691c6f4","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:00:18.360Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c260206d-0b8d-42e0-a178-e3377691c6f4","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:18.360Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c260206d-0b8d-42e0-a178-e3377691c6f4"}
{"level":"error","ts":"2026-09-18T10:00:18.360Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c260206d-0b8d-42e0-a178-e3377691c6f4","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:19.363Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"834ea608-1066-4798-a87c-d16a5715167d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:19.363Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"834ea608-1066-4798-a87c-d16a5715167d","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:19.363Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"834ea608-1066-4798-a87c-d16a5715167d","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:19.363Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"834ea608-1066-4798-a87c-d16a5715167d"}
{"level":"info","ts":"2026-09-18T10:00:19.363Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"834ea608-1066-4798-a87c-d16a5715167d","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:00:19.368Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"834ea608-1066-4798-a87c-d16a5715167d","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:19.368Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"834ea608-1066-4798-a87c-d16a5715167d"}
{"level":"error","ts":"2026-09-18T10:00:19.368Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"834ea608-1066-4798-a87c-d16a5715167d","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:19.378Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a7ca7b7c-e80d-4925-81eb-73759d909a7a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:19.378Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a7ca7b7c-e80d-4925-81eb-73759d909a7a","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:19.378Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a7ca7b7c-e80d-4925-81eb-73759d909a7a","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:19.378Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a7ca7b7c-e80d-4925-81eb-73759d909a7a"}
{"level":"info","ts":"2026-09-18T10:00:19.378Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a7ca7b7c-e80d-4925-81eb-73759d909a7a","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:00:19.383Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a7ca7b7c-e80d-4925-81eb-73759d909a7a","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:19.383Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a7ca7b7c-e80d-4925-81eb-73759d909a7a"}
{"level":"error","ts":"2026-09-18T10:00:19.383Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"a7ca7b7c-e80d-4925-81eb-73759d909a7a","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:31.069Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"87cd33eb-1284-4ee1-a053-1385a6989b0d","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:31.069Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"87cd33eb-1284-4ee1-a053-1385a6989b0d","pod":"fail-k8s-job-q64ml-head-clvdq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-09-18T10:00:31.069Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"87cd33eb-1284-4ee1-a053-1385a6989b0d","pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","ip":"10.132.0.41"}
{"level":"info","ts":"2026-09-18T10:00:31.069Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"87cd33eb-1284-4ee1-a053-1385a6989b0d","rayCluster":"fail-k8s-job-q64ml","podIPs":["10.133.0.40","10.132.0.41"]}
{"level":"info","ts":"2026-09-18T10:00:36.023Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d3213338-9117-4308-a8d0-2e2069708191","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:36.023Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d3213338-9117-4308-a8d0-2e2069708191","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:36.023Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d3213338-9117-4308-a8d0-2e2069708191","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:36.023Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d3213338-9117-4308-a8d0-2e2069708191"}
{"level":"info","ts":"2026-09-18T10:00:36.023Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d3213338-9117-4308-a8d0-2e2069708191","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:00:36.031Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d3213338-9117-4308-a8d0-2e2069708191","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:36.031Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d3213338-9117-4308-a8d0-2e2069708191"}
{"level":"error","ts":"2026-09-18T10:00:36.031Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"d3213338-9117-4308-a8d0-2e2069708191","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:38.448Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"cbe21601-5434-487c-8b66-c6a60efa30c7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:38.448Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"cbe21601-5434-487c-8b66-c6a60efa30c7","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:38.448Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"cbe21601-5434-487c-8b66-c6a60efa30c7","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:38.448Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"cbe21601-5434-487c-8b66-c6a60efa30c7"}
{"level":"info","ts":"2026-09-18T10:00:38.448Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"cbe21601-5434-487c-8b66-c6a60efa30c7","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:00:38.453Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"cbe21601-5434-487c-8b66-c6a60efa30c7","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:38.453Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"cbe21601-5434-487c-8b66-c6a60efa30c7"}
{"level":"error","ts":"2026-09-18T10:00:38.453Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"cbe21601-5434-487c-8b66-c6a60efa30c7","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:41.465Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"46874032-00b0-463e-abfe-c51c69b5db1f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:41.465Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"46874032-00b0-463e-abfe-c51c69b5db1f","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:41.465Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"46874032-00b0-463e-abfe-c51c69b5db1f","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:41.465Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"46874032-00b0-463e-abfe-c51c69b5db1f"}
{"level":"info","ts":"2026-09-18T10:00:41.465Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"46874032-00b0-463e-abfe-c51c69b5db1f","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:00:41.470Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"46874032-00b0-463e-abfe-c51c69b5db1f","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:41.470Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"46874032-00b0-463e-abfe-c51c69b5db1f"}
{"level":"error","ts":"2026-09-18T10:00:41.470Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"46874032-00b0-463e-abfe-c51c69b5db1f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:42.473Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"acc1bee8-9ec4-4921-a8a8-abc6ff9ac63c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:42.473Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"acc1bee8-9ec4-4921-a8a8-abc6ff9ac63c","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:42.473Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"acc1bee8-9ec4-4921-a8a8-abc6ff9ac63c","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:42.473Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"acc1bee8-9ec4-4921-a8a8-abc6ff9ac63c"}
{"level":"info","ts":"2026-09-18T10:00:42.473Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"acc1bee8-9ec4-4921-a8a8-abc6ff9ac63c","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:00:42.478Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"acc1bee8-9ec4-4921-a8a8-abc6ff9ac63c","JobId":"fail-k8s-job-m2v6f","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:42.479Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"acc1bee8-9ec4-4921-a8a8-abc6ff9ac63c"}
{"level":"error","ts":"2026-09-18T10:00:42.479Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"acc1bee8-9ec4-4921-a8a8-abc6ff9ac63c","error":"Job fail-k8s-job-m2v6f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:42.492Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f557dd23-903d-4612-ac8a-a883f92adf01","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:42.492Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f557dd23-903d-4612-ac8a-a883f92adf01","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:42.492Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f557dd23-903d-4612-ac8a-a883f92adf01","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:42.492Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f557dd23-903d-4612-ac8a-a883f92adf01"}
{"level":"info","ts":"2026-09-18T10:00:42.492Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f557dd23-903d-4612-ac8a-a883f92adf01","rayCluster":"fail-k8s-job-q64ml","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:00:42.492Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has failed. Attempting to transition the status to `Failed`.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f557dd23-903d-4612-ac8a-a883f92adf01","Submitter K8s Job":"fail-k8s-job","Reason":"BackoffLimitExceeded","Message":"Job has reached the specified backoff limit"}
{"level":"info","ts":"2026-09-18T10:00:42.492Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f557dd23-903d-4612-ac8a-a883f92adf01","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","dashboardURL":"fail-k8s-job-q64ml-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-m2v6f","rayClusterName":"fail-k8s-job-q64ml","dashboardURL":"fail-k8s-job-q64ml-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Failed","reason":"SubmissionFailed","message":"Job submission has failed. Reason: BackoffLimitExceeded. Message: Job has reached the specified backoff limit","startTime":"2026-09-18T09:58:00Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T09:59:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"fail-k8s-job-q64ml-head-clvdq","serviceName":"fail-k8s-job-q64ml-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T09:58:21Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T09:58:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T09:58:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:42.492Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"f557dd23-903d-4612-ac8a-a883f92adf01","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-09-18T10:00:42.508Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c0777c3d-ca97-4eb8-a6dd-6663bfaff77a","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:42.508Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c0777c3d-ca97-4eb8-a6dd-6663bfaff77a","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:00:42 +0000 UTC","Now":"2026-09-18T10:00:42.508Z","ShutdownTime":"2026-09-18T10:00:42.000Z"}
{"level":"info","ts":"2026-09-18T10:00:42.517Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c0777c3d-ca97-4eb8-a6dd-6663bfaff77a","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:42.517Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c0777c3d-ca97-4eb8-a6dd-6663bfaff77a","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T10:00:42.517Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"c0777c3d-ca97-4eb8-a6dd-6663bfaff77a","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:42.518Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"reconcileID":"612b188c-362f-4f43-aa64-c6321275d02a"}
{"level":"info","ts":"2026-09-18T10:00:42.518Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"c2aaa0ad-a7b4-46a6-a12d-9ec7d605d445","rayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:42.518Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"74690922-4dc7-47ab-8c75-76bc917c1d1f","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:42.518Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"74690922-4dc7-47ab-8c75-76bc917c1d1f","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:00:42 +0000 UTC","Now":"2026-09-18T10:00:42.518Z","ShutdownTime":"2026-09-18T10:00:42.000Z"}
{"level":"info","ts":"2026-09-18T10:00:42.518Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"74690922-4dc7-47ab-8c75-76bc917c1d1f","RayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:42.518Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"74690922-4dc7-47ab-8c75-76bc917c1d1f","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T10:00:42.518Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"74690922-4dc7-47ab-8c75-76bc917c1d1f","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:42.518Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"c2aaa0ad-a7b4-46a6-a12d-9ec7d605d445","pod":"fail-k8s-job-q64ml-head-clvdq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-09-18T10:00:42.518Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"c2aaa0ad-a7b4-46a6-a12d-9ec7d605d445","pod":"fail-k8s-job-q64ml-small-group-worker-9sjpx","ip":"10.132.0.41"}
{"level":"info","ts":"2026-09-18T10:00:42.518Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"c502818d-d1ae-43fa-a3cb-f127c4b4716c","namespacedName":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:42.518Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"c502818d-d1ae-43fa-a3cb-f127c4b4716c","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:42.518Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"c502818d-d1ae-43fa-a3cb-f127c4b4716c","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:42.518Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"c2aaa0ad-a7b4-46a6-a12d-9ec7d605d445","rayCluster":"fail-k8s-job-q64ml","podIPs":["10.133.0.40","10.132.0.41"]}
{"level":"info","ts":"2026-09-18T10:00:42.518Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"c502818d-d1ae-43fa-a3cb-f127c4b4716c","namespace":"test-ns-ltx56","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T10:00:42.518Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"5d0811b5-c580-4f44-af59-630aac8ddfdd"}
{"level":"info","ts":"2026-09-18T10:00:42.525Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"c502818d-d1ae-43fa-a3cb-f127c4b4716c","name":"kuberay-gateway-access","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:00:42.531Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"c502818d-d1ae-43fa-a3cb-f127c4b4716c","name":"test-ns-ltx56-fail-k8s-job-q64ml","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T10:00:42.537Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"c502818d-d1ae-43fa-a3cb-f127c4b4716c","configMap":"kube-rbac-proxy-config-fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:42.542Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"c502818d-d1ae-43fa-a3cb-f127c4b4716c","serviceAccount":"fail-k8s-job-q64ml-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:00:42.570Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:42.570Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:00:42.581Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"c502818d-d1ae-43fa-a3cb-f127c4b4716c","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:42.581Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"11574fbf-8654-4ba1-b4f9-cb45b05d829b","namespacedName":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:42.581Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"11574fbf-8654-4ba1-b4f9-cb45b05d829b","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:42.581Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"11574fbf-8654-4ba1-b4f9-cb45b05d829b","cluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:42.582Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"11574fbf-8654-4ba1-b4f9-cb45b05d829b","namespace":"test-ns-ltx56","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T10:00:42.582Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"11574fbf-8654-4ba1-b4f9-cb45b05d829b","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-09-18T10:00:42.582Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"79cb0889-13a3-41e2-a47c-302c7ab2e82d","namespace":"test-ns-ltx56","clusterName":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:42.582Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b605ebe3-61ed-4dc9-8088-35ee69482d22","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:42.582Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b605ebe3-61ed-4dc9-8088-35ee69482d22","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:00:42 +0000 UTC","Now":"2026-09-18T10:00:42.582Z","ShutdownTime":"2026-09-18T10:00:42.000Z"}
{"level":"info","ts":"2026-09-18T10:00:42.582Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b605ebe3-61ed-4dc9-8088-35ee69482d22","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:42.582Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b605ebe3-61ed-4dc9-8088-35ee69482d22","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T10:00:42.582Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"b605ebe3-61ed-4dc9-8088-35ee69482d22","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:42.582Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"8b005e65-9ec7-4a9a-9a3e-bc2603d8cf75"}
{"level":"error","ts":"2026-09-18T10:00:42.587Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"11574fbf-8654-4ba1-b4f9-cb45b05d829b","error":"rayclusters.ray.io \"fail-k8s-job-q64ml\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:196\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:137\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:42.587Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"11574fbf-8654-4ba1-b4f9-cb45b05d829b"}
{"level":"error","ts":"2026-09-18T10:00:42.587Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"11574fbf-8654-4ba1-b4f9-cb45b05d829b","error":"rayclusters.ray.io \"fail-k8s-job-q64ml\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:42.587Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"150c28d9-4d03-4824-8027-fa23f212d824","namespacedName":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:42.587Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"150c28d9-4d03-4824-8027-fa23f212d824"}
{"level":"info","ts":"2026-09-18T10:00:42.587Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"150c28d9-4d03-4824-8027-fa23f212d824","namespace":"test-ns-ltx56","authClusters":0}
{"level":"info","ts":"2026-09-18T10:00:42.587Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"150c28d9-4d03-4824-8027-fa23f212d824","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:00:42.592Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"3d4b8079-76a5-470e-b748-6a734df7c87d","namespacedName":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:42.592Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"3d4b8079-76a5-470e-b748-6a734df7c87d"}
{"level":"info","ts":"2026-09-18T10:00:42.592Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"3d4b8079-76a5-470e-b748-6a734df7c87d","namespace":"test-ns-ltx56","authClusters":0}
{"level":"info","ts":"2026-09-18T10:00:42.592Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"3d4b8079-76a5-470e-b748-6a734df7c87d","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:00:42.592Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"2fa12672-b7d8-4a0a-8cf5-539cce51c3d6"}
{"level":"info","ts":"2026-09-18T10:00:42.600Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"e7df20dc-a9d6-43b8-a791-ee6b2f497eca"}
{"level":"info","ts":"2026-09-18T10:00:42.603Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"ca66044c-6b16-442a-87d2-47b98f8ed4e4","namespacedName":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:42.603Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"ca66044c-6b16-442a-87d2-47b98f8ed4e4"}
{"level":"info","ts":"2026-09-18T10:00:42.665Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"ca66044c-6b16-442a-87d2-47b98f8ed4e4","namespace":"test-ns-ltx56","authClusters":0}
{"level":"info","ts":"2026-09-18T10:00:42.666Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"ca66044c-6b16-442a-87d2-47b98f8ed4e4","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:00:42.678Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"79cb0889-13a3-41e2-a47c-302c7ab2e82d","namespace":"test-ns-ltx56","clusterName":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:00:42.825Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"0d9b6de2-aba6-491b-9ee6-1a319b1627fd","DeletionTimestamp":"2026-09-18 10:00:42 +0000 UTC"}
{"level":"error","ts":"2026-09-18T10:00:42.825Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"0d9b6de2-aba6-491b-9ee6-1a319b1627fd","error":"RayCluster.ray.io \"fail-k8s-job-q64ml\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:121\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:00:43.654Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"0d9b6de2-aba6-491b-9ee6-1a319b1627fd","error":"Post \"http://fail-k8s-job-q64ml-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/fail-k8s-job-m2v6f/stop\": EOF","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:43.654Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"0d9b6de2-aba6-491b-9ee6-1a319b1627fd","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T10:00:43.669Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f5b918bc-5c51-48ed-8340-4c588e21ab83","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:43.669Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f5b918bc-5c51-48ed-8340-4c588e21ab83","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T10:00:43.683Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f5b918bc-5c51-48ed-8340-4c588e21ab83"}
{"level":"info","ts":"2026-09-18T10:00:43.683Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f5b918bc-5c51-48ed-8340-4c588e21ab83","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T10:00:43.683Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f5b918bc-5c51-48ed-8340-4c588e21ab83","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:00:43.683Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f5b918bc-5c51-48ed-8340-4c588e21ab83","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:00:43.694Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"9b0ec183-8d58-409d-9c6b-9a6d7d27620f"}
{"level":"info","ts":"2026-09-18T10:00:43.695Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"45e14ce1-d595-4963-b4fb-973b278cfd98","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:43.695Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"45e14ce1-d595-4963-b4fb-973b278cfd98"}
{"level":"info","ts":"2026-09-18T10:00:43.695Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"45e14ce1-d595-4963-b4fb-973b278cfd98","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T10:00:43.695Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"45e14ce1-d595-4963-b4fb-973b278cfd98","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-gxhst","rayClusterName":"stop-xqbjp","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:00:43.695Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"45e14ce1-d595-4963-b4fb-973b278cfd98","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:00:43.702Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"45e14ce1-d595-4963-b4fb-973b278cfd98","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T10:00:43.702Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"45e14ce1-d595-4963-b4fb-973b278cfd98"}
{"level":"error","ts":"2026-09-18T10:00:43.702Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"45e14ce1-d595-4963-b4fb-973b278cfd98","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:43.703Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"42b03028-3f06-46bc-9217-a072ba947443","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:43.703Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"42b03028-3f06-46bc-9217-a072ba947443","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.703Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"42b03028-3f06-46bc-9217-a072ba947443","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:43.703Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"42b03028-3f06-46bc-9217-a072ba947443","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:43.708Z","logger":"raycluster-resource","msg":"default","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.708Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-9cztm","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:00:43.726Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"42b03028-3f06-46bc-9217-a072ba947443","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:43.726Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"7fb9edd9-5e91-4be2-a651-d895757a6f4f","namespacedName":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:43.726Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"7fb9edd9-5e91-4be2-a651-d895757a6f4f","mode":"IntegratedOAuth","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.726Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"7fb9edd9-5e91-4be2-a651-d895757a6f4f","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.726Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"7fb9edd9-5e91-4be2-a651-d895757a6f4f","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"42b03028-3f06-46bc-9217-a072ba947443"}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"42b03028-3f06-46bc-9217-a072ba947443","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"42b03028-3f06-46bc-9217-a072ba947443","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bdc17546-4b80-4932-afd2-b2a556c306ed","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bdc17546-4b80-4932-afd2-b2a556c306ed","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bdc17546-4b80-4932-afd2-b2a556c306ed","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"f0f9bf2d-8c6e-4aea-acbe-3213f8dd97f9"}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"f0f9bf2d-8c6e-4aea-acbe-3213f8dd97f9"}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"f0f9bf2d-8c6e-4aea-acbe-3213f8dd97f9"}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bdc17546-4b80-4932-afd2-b2a556c306ed"}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bdc17546-4b80-4932-afd2-b2a556c306ed","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bdc17546-4b80-4932-afd2-b2a556c306ed","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fc0485ab-b09d-41dd-bec1-7aada70d84ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fc0485ab-b09d-41dd-bec1-7aada70d84ff","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fc0485ab-b09d-41dd-bec1-7aada70d84ff","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"0e587493-5abb-4882-bb63-984c42105726","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"0e587493-5abb-4882-bb63-984c42105726","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.727Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fc0485ab-b09d-41dd-bec1-7aada70d84ff"}
{"level":"info","ts":"2026-09-18T10:00:43.728Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fc0485ab-b09d-41dd-bec1-7aada70d84ff","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:43.728Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fc0485ab-b09d-41dd-bec1-7aada70d84ff","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:00:43.728Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"b390c32d-9a6d-43f2-8250-044affe78bb8","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.766Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"f0f9bf2d-8c6e-4aea-acbe-3213f8dd97f9","name":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:43.766Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"f0f9bf2d-8c6e-4aea-acbe-3213f8dd97f9","error":"head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"error","ts":"2026-09-18T10:00:43.766Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"f0f9bf2d-8c6e-4aea-acbe-3213f8dd97f9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:43.766Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"f0f9bf2d-8c6e-4aea-acbe-3213f8dd97f9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"info","ts":"2026-09-18T10:00:43.766Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"f0f9bf2d-8c6e-4aea-acbe-3213f8dd97f9","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:43.766Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"f0f9bf2d-8c6e-4aea-acbe-3213f8dd97f9","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:43.766Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"f0f9bf2d-8c6e-4aea-acbe-3213f8dd97f9","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:43.766Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"f0f9bf2d-8c6e-4aea-acbe-3213f8dd97f9","name":"stop-9cztm","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:00:43.767Z","logger":"raycluster-resource","msg":"default","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.767Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-9cztm","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:00:43.773Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"b390c32d-9a6d-43f2-8250-044affe78bb8","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.773Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"59489d8a-693a-4cc7-9891-2e5c608ef7bd","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.779Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"f0f9bf2d-8c6e-4aea-acbe-3213f8dd97f9"}
{"level":"error","ts":"2026-09-18T10:00:43.779Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"f0f9bf2d-8c6e-4aea-acbe-3213f8dd97f9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:43.779Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d03ac4dc-a7a3-4063-9516-6ebf3fee07ea"}
{"level":"info","ts":"2026-09-18T10:00:43.779Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d03ac4dc-a7a3-4063-9516-6ebf3fee07ea"}
{"level":"info","ts":"2026-09-18T10:00:43.779Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d03ac4dc-a7a3-4063-9516-6ebf3fee07ea"}
{"level":"info","ts":"2026-09-18T10:00:43.779Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d03ac4dc-a7a3-4063-9516-6ebf3fee07ea","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:43.779Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d03ac4dc-a7a3-4063-9516-6ebf3fee07ea","error":"head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"error","ts":"2026-09-18T10:00:43.779Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d03ac4dc-a7a3-4063-9516-6ebf3fee07ea","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:43.779Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d03ac4dc-a7a3-4063-9516-6ebf3fee07ea","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"info","ts":"2026-09-18T10:00:43.779Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d03ac4dc-a7a3-4063-9516-6ebf3fee07ea","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:43.780Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d03ac4dc-a7a3-4063-9516-6ebf3fee07ea","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:43.780Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d03ac4dc-a7a3-4063-9516-6ebf3fee07ea","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:43.780Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d03ac4dc-a7a3-4063-9516-6ebf3fee07ea","name":"stop-9cztm","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:00:43.866Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"316ac1c6-8723-4da1-bd93-2beb14e9a18b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:43.866Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"316ac1c6-8723-4da1-bd93-2beb14e9a18b","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.866Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"316ac1c6-8723-4da1-bd93-2beb14e9a18b","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:43.866Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"316ac1c6-8723-4da1-bd93-2beb14e9a18b"}
{"level":"info","ts":"2026-09-18T10:00:43.866Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"316ac1c6-8723-4da1-bd93-2beb14e9a18b","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:43.866Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"316ac1c6-8723-4da1-bd93-2beb14e9a18b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:43.866Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"316ac1c6-8723-4da1-bd93-2beb14e9a18b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:00:43.866Z","logger":"raycluster-resource","msg":"default","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.867Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-9cztm","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:00:43.867Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d03ac4dc-a7a3-4063-9516-6ebf3fee07ea","name":"stop-9cztm","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-9cztm\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"stop-9cztm"}}
{"level":"info","ts":"2026-09-18T10:00:43.867Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d03ac4dc-a7a3-4063-9516-6ebf3fee07ea"}
{"level":"error","ts":"2026-09-18T10:00:43.867Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d03ac4dc-a7a3-4063-9516-6ebf3fee07ea","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:43.867Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"6e920335-0092-45db-946c-42c716983ce0"}
{"level":"info","ts":"2026-09-18T10:00:43.867Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"6e920335-0092-45db-946c-42c716983ce0"}
{"level":"info","ts":"2026-09-18T10:00:43.867Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"6e920335-0092-45db-946c-42c716983ce0"}
{"level":"info","ts":"2026-09-18T10:00:43.867Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"6e920335-0092-45db-946c-42c716983ce0","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:43.867Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"6e920335-0092-45db-946c-42c716983ce0","error":"head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"error","ts":"2026-09-18T10:00:43.867Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"6e920335-0092-45db-946c-42c716983ce0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:43.867Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"6e920335-0092-45db-946c-42c716983ce0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"info","ts":"2026-09-18T10:00:43.867Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"6e920335-0092-45db-946c-42c716983ce0","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:43.868Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"0e587493-5abb-4882-bb63-984c42105726","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.868Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"0e587493-5abb-4882-bb63-984c42105726","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.868Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"6e920335-0092-45db-946c-42c716983ce0","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:43.868Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"6e920335-0092-45db-946c-42c716983ce0","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:43.868Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"6e920335-0092-45db-946c-42c716983ce0"}
{"level":"error","ts":"2026-09-18T10:00:43.868Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"6e920335-0092-45db-946c-42c716983ce0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:00:43.871Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"7fb9edd9-5e91-4be2-a651-d895757a6f4f","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"stop-9cztm\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:43.871Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"7fb9edd9-5e91-4be2-a651-d895757a6f4f"}
{"level":"error","ts":"2026-09-18T10:00:43.871Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"7fb9edd9-5e91-4be2-a651-d895757a6f4f","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"stop-9cztm\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:43.871Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"59fb72d8-cbff-49ee-92cb-13c67eda64c7","namespacedName":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:43.871Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"59fb72d8-cbff-49ee-92cb-13c67eda64c7","mode":"IntegratedOAuth","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.871Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"59fb72d8-cbff-49ee-92cb-13c67eda64c7","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.871Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"59fb72d8-cbff-49ee-92cb-13c67eda64c7","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.872Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"59489d8a-693a-4cc7-9891-2e5c608ef7bd","name":"stop-9cztm-head"}
{"level":"info","ts":"2026-09-18T10:00:43.877Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"29c1aff7-c4ac-4716-8f81-3d050a19a218"}
{"level":"info","ts":"2026-09-18T10:00:43.877Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"29c1aff7-c4ac-4716-8f81-3d050a19a218"}
{"level":"info","ts":"2026-09-18T10:00:43.877Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"29c1aff7-c4ac-4716-8f81-3d050a19a218"}
{"level":"info","ts":"2026-09-18T10:00:43.877Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"29c1aff7-c4ac-4716-8f81-3d050a19a218","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:43.878Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"29c1aff7-c4ac-4716-8f81-3d050a19a218","error":"head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"error","ts":"2026-09-18T10:00:43.878Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"29c1aff7-c4ac-4716-8f81-3d050a19a218","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:43.878Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"29c1aff7-c4ac-4716-8f81-3d050a19a218","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"info","ts":"2026-09-18T10:00:43.878Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"29c1aff7-c4ac-4716-8f81-3d050a19a218","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:43.878Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"29c1aff7-c4ac-4716-8f81-3d050a19a218","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:43.878Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"29c1aff7-c4ac-4716-8f81-3d050a19a218","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:43.878Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"29c1aff7-c4ac-4716-8f81-3d050a19a218"}
{"level":"error","ts":"2026-09-18T10:00:43.878Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"29c1aff7-c4ac-4716-8f81-3d050a19a218","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:43.878Z","logger":"raycluster-resource","msg":"default","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.878Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-9cztm","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:00:43.881Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"3c3c852e-0246-4074-be2f-55d0f88a6dd2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:43.881Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"3c3c852e-0246-4074-be2f-55d0f88a6dd2","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.881Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"3c3c852e-0246-4074-be2f-55d0f88a6dd2","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:43.881Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"3c3c852e-0246-4074-be2f-55d0f88a6dd2"}
{"level":"info","ts":"2026-09-18T10:00:43.881Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"3c3c852e-0246-4074-be2f-55d0f88a6dd2","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:43.881Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"3c3c852e-0246-4074-be2f-55d0f88a6dd2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:43.882Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"0e587493-5abb-4882-bb63-984c42105726","rayCluster":"stop-9cztm","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T10:00:43.882Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"0e587493-5abb-4882-bb63-984c42105726","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.966Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3675a41-eecb-4cbf-a640-c4222d7ec9bd"}
{"level":"info","ts":"2026-09-18T10:00:43.966Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3675a41-eecb-4cbf-a640-c4222d7ec9bd"}
{"level":"info","ts":"2026-09-18T10:00:43.966Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3675a41-eecb-4cbf-a640-c4222d7ec9bd"}
{"level":"info","ts":"2026-09-18T10:00:43.966Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3675a41-eecb-4cbf-a640-c4222d7ec9bd","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:43.966Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3675a41-eecb-4cbf-a640-c4222d7ec9bd","error":"head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"error","ts":"2026-09-18T10:00:43.966Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3675a41-eecb-4cbf-a640-c4222d7ec9bd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:43.966Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3675a41-eecb-4cbf-a640-c4222d7ec9bd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"info","ts":"2026-09-18T10:00:43.966Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3675a41-eecb-4cbf-a640-c4222d7ec9bd","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:43.966Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3675a41-eecb-4cbf-a640-c4222d7ec9bd","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:43.966Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3675a41-eecb-4cbf-a640-c4222d7ec9bd","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:43.966Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3675a41-eecb-4cbf-a640-c4222d7ec9bd"}
{"level":"error","ts":"2026-09-18T10:00:43.966Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3675a41-eecb-4cbf-a640-c4222d7ec9bd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:43.967Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"0e587493-5abb-4882-bb63-984c42105726","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.967Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"67f25eca-ca75-4631-b9b2-fba24f0971e2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:43.967Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"67f25eca-ca75-4631-b9b2-fba24f0971e2","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.967Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"67f25eca-ca75-4631-b9b2-fba24f0971e2","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:43.967Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"67f25eca-ca75-4631-b9b2-fba24f0971e2"}
{"level":"info","ts":"2026-09-18T10:00:43.967Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"67f25eca-ca75-4631-b9b2-fba24f0971e2","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:43.967Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"67f25eca-ca75-4631-b9b2-fba24f0971e2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:43.974Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"59fb72d8-cbff-49ee-92cb-13c67eda64c7","name":"stop-9cztm-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:00:43.974Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"59489d8a-693a-4cc7-9891-2e5c608ef7bd","name":"stop-9cztm-workers"}
{"level":"info","ts":"2026-09-18T10:00:43.974Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"59489d8a-693a-4cc7-9891-2e5c608ef7bd","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.975Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"f2eb362e-e4d7-43f9-8728-70ce11529bb7","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:43.991Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"0e587493-5abb-4882-bb63-984c42105726","rayCluster":"stop-9cztm","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:00:43.991Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"59fb72d8-cbff-49ee-92cb-13c67eda64c7","name":"kuberay-gateway-access","namespace":"test-ns-ltx56","operation":"created"}
{"level":"info","ts":"2026-09-18T10:00:43.996Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"f2eb362e-e4d7-43f9-8728-70ce11529bb7","name":"stop-9cztm-head"}
{"level":"info","ts":"2026-09-18T10:00:44.066Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a7317fa0-7434-4e68-b107-894e6324b888"}
{"level":"info","ts":"2026-09-18T10:00:44.066Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a7317fa0-7434-4e68-b107-894e6324b888"}
{"level":"info","ts":"2026-09-18T10:00:44.066Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a7317fa0-7434-4e68-b107-894e6324b888"}
{"level":"info","ts":"2026-09-18T10:00:44.066Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a7317fa0-7434-4e68-b107-894e6324b888","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:44.066Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a7317fa0-7434-4e68-b107-894e6324b888","error":"head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"error","ts":"2026-09-18T10:00:44.066Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a7317fa0-7434-4e68-b107-894e6324b888","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:44.066Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a7317fa0-7434-4e68-b107-894e6324b888","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"info","ts":"2026-09-18T10:00:44.066Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a7317fa0-7434-4e68-b107-894e6324b888","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:44.066Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a7317fa0-7434-4e68-b107-894e6324b888","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:44.066Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a7317fa0-7434-4e68-b107-894e6324b888","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:44.066Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a7317fa0-7434-4e68-b107-894e6324b888"}
{"level":"error","ts":"2026-09-18T10:00:44.066Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a7317fa0-7434-4e68-b107-894e6324b888","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:44.066Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"59fb72d8-cbff-49ee-92cb-13c67eda64c7","name":"test-ns-ltx56-stop-9cztm","operation":"created"}
{"level":"info","ts":"2026-09-18T10:00:44.067Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"0e587493-5abb-4882-bb63-984c42105726","rayCluster":"stop-9cztm","podIPs":[]}
{"level":"error","ts":"2026-09-18T10:00:44.067Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"0e587493-5abb-4882-bb63-984c42105726","error":"Certificate.cert-manager.io \"ray-worker-cert-stop-9cztm\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:44.067Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"0e587493-5abb-4882-bb63-984c42105726"}
{"level":"error","ts":"2026-09-18T10:00:44.067Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"0e587493-5abb-4882-bb63-984c42105726","error":"Certificate.cert-manager.io \"ray-worker-cert-stop-9cztm\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:44.067Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"734f389c-2753-4a2f-a222-4349617f684b","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:44.082Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"f2eb362e-e4d7-43f9-8728-70ce11529bb7","name":"stop-9cztm-workers"}
{"level":"info","ts":"2026-09-18T10:00:44.082Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"f2eb362e-e4d7-43f9-8728-70ce11529bb7","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:44.082Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"59fb72d8-cbff-49ee-92cb-13c67eda64c7","name":"kube-rbac-proxy-config-stop-9cztm","operation":"created"}
{"level":"info","ts":"2026-09-18T10:00:44.095Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"59fb72d8-cbff-49ee-92cb-13c67eda64c7","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:44.095Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"1ba58f0e-8ee8-42f3-91c2-adc89ebbedb9","namespacedName":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:44.095Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"1ba58f0e-8ee8-42f3-91c2-adc89ebbedb9","mode":"IntegratedOAuth","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:44.095Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"1ba58f0e-8ee8-42f3-91c2-adc89ebbedb9","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:44.095Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"7070071f-d080-4874-9380-46a467840ef2","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:44.096Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6017b6b5-317e-4261-9c25-491dd2396bb4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:44.096Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6017b6b5-317e-4261-9c25-491dd2396bb4","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:44.096Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6017b6b5-317e-4261-9c25-491dd2396bb4","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:44.096Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6017b6b5-317e-4261-9c25-491dd2396bb4"}
{"level":"info","ts":"2026-09-18T10:00:44.096Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6017b6b5-317e-4261-9c25-491dd2396bb4","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:44.096Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6017b6b5-317e-4261-9c25-491dd2396bb4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:44.096Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6017b6b5-317e-4261-9c25-491dd2396bb4","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"error","ts":"2026-09-18T10:00:44.097Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"734f389c-2753-4a2f-a222-4349617f684b","error":"certificates.cert-manager.io \"ray-worker-cert-stop-9cztm\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:00:44.097Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"734f389c-2753-4a2f-a222-4349617f684b","error":"certificates.cert-manager.io \"ray-worker-cert-stop-9cztm\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:44.097Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"734f389c-2753-4a2f-a222-4349617f684b"}
{"level":"error","ts":"2026-09-18T10:00:44.097Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"734f389c-2753-4a2f-a222-4349617f684b","error":"certificates.cert-manager.io \"ray-worker-cert-stop-9cztm\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:44.097Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"ce0d65cc-61b7-4562-8f72-ad27c46870b0","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:44.097Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"ce0d65cc-61b7-4562-8f72-ad27c46870b0"}
{"level":"info","ts":"2026-09-18T10:00:44.107Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"f3ff0025-8e92-4643-b13a-74baa10fa50e","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:44.107Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"f3ff0025-8e92-4643-b13a-74baa10fa50e"}
{"level":"info","ts":"2026-09-18T10:00:44.108Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"7070071f-d080-4874-9380-46a467840ef2","name":"stop-9cztm-head"}
{"level":"info","ts":"2026-09-18T10:00:44.167Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"406fe9a3-e6f7-448d-9488-876f0ace63a9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:44.167Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"406fe9a3-e6f7-448d-9488-876f0ace63a9","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:44.167Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"406fe9a3-e6f7-448d-9488-876f0ace63a9","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:44.167Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"406fe9a3-e6f7-448d-9488-876f0ace63a9"}
{"level":"info","ts":"2026-09-18T10:00:44.167Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"406fe9a3-e6f7-448d-9488-876f0ace63a9","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:44.167Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"406fe9a3-e6f7-448d-9488-876f0ace63a9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:44.190Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"7070071f-d080-4874-9380-46a467840ef2","name":"stop-9cztm-workers"}
{"level":"info","ts":"2026-09-18T10:00:44.190Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"7070071f-d080-4874-9380-46a467840ef2","cluster":"stop-9cztm"}
{"level":"error","ts":"2026-09-18T10:00:44.190Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"1ba58f0e-8ee8-42f3-91c2-adc89ebbedb9","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-9cztm\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:44.190Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"1ba58f0e-8ee8-42f3-91c2-adc89ebbedb9"}
{"level":"error","ts":"2026-09-18T10:00:44.190Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"1ba58f0e-8ee8-42f3-91c2-adc89ebbedb9","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-9cztm\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:44.195Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"b6f261bc-ac62-4442-8f72-2f135e2b8800","namespacedName":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:44.195Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"b6f261bc-ac62-4442-8f72-2f135e2b8800","mode":"IntegratedOAuth","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:44.195Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"b6f261bc-ac62-4442-8f72-2f135e2b8800","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:44.206Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"b6f261bc-ac62-4442-8f72-2f135e2b8800","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:44.228Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"8654c181-180a-4361-a677-1735ca8f11b1"}
{"level":"info","ts":"2026-09-18T10:00:44.228Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"8654c181-180a-4361-a677-1735ca8f11b1"}
{"level":"info","ts":"2026-09-18T10:00:44.228Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"8654c181-180a-4361-a677-1735ca8f11b1"}
{"level":"info","ts":"2026-09-18T10:00:44.228Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"8654c181-180a-4361-a677-1735ca8f11b1","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:44.228Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"8654c181-180a-4361-a677-1735ca8f11b1","error":"head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"error","ts":"2026-09-18T10:00:44.228Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"8654c181-180a-4361-a677-1735ca8f11b1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:44.228Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"8654c181-180a-4361-a677-1735ca8f11b1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"info","ts":"2026-09-18T10:00:44.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"8654c181-180a-4361-a677-1735ca8f11b1","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:44.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"8654c181-180a-4361-a677-1735ca8f11b1","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:44.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"8654c181-180a-4361-a677-1735ca8f11b1","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:44.228Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"8654c181-180a-4361-a677-1735ca8f11b1"}
{"level":"error","ts":"2026-09-18T10:00:44.228Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"8654c181-180a-4361-a677-1735ca8f11b1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:44.549Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"86269b24-c9e8-4dfc-bc01-5db37d9f5ac8"}
{"level":"info","ts":"2026-09-18T10:00:44.549Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"86269b24-c9e8-4dfc-bc01-5db37d9f5ac8"}
{"level":"info","ts":"2026-09-18T10:00:44.549Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"86269b24-c9e8-4dfc-bc01-5db37d9f5ac8"}
{"level":"info","ts":"2026-09-18T10:00:44.549Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"86269b24-c9e8-4dfc-bc01-5db37d9f5ac8","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:44.549Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"86269b24-c9e8-4dfc-bc01-5db37d9f5ac8","error":"head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"error","ts":"2026-09-18T10:00:44.549Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"86269b24-c9e8-4dfc-bc01-5db37d9f5ac8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:44.549Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"86269b24-c9e8-4dfc-bc01-5db37d9f5ac8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"info","ts":"2026-09-18T10:00:44.549Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"86269b24-c9e8-4dfc-bc01-5db37d9f5ac8","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:44.549Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"86269b24-c9e8-4dfc-bc01-5db37d9f5ac8","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:44.549Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"86269b24-c9e8-4dfc-bc01-5db37d9f5ac8","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:44.549Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"86269b24-c9e8-4dfc-bc01-5db37d9f5ac8"}
{"level":"error","ts":"2026-09-18T10:00:44.549Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"86269b24-c9e8-4dfc-bc01-5db37d9f5ac8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:45.190Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"69811076-6ccf-42fb-b736-92a97ccdafed"}
{"level":"info","ts":"2026-09-18T10:00:45.190Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"69811076-6ccf-42fb-b736-92a97ccdafed"}
{"level":"info","ts":"2026-09-18T10:00:45.190Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"69811076-6ccf-42fb-b736-92a97ccdafed"}
{"level":"info","ts":"2026-09-18T10:00:45.191Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"69811076-6ccf-42fb-b736-92a97ccdafed","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:45.191Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"69811076-6ccf-42fb-b736-92a97ccdafed","error":"head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"error","ts":"2026-09-18T10:00:45.191Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"69811076-6ccf-42fb-b736-92a97ccdafed","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:45.191Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"69811076-6ccf-42fb-b736-92a97ccdafed","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found"}
{"level":"info","ts":"2026-09-18T10:00:45.191Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"69811076-6ccf-42fb-b736-92a97ccdafed","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:45.191Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"69811076-6ccf-42fb-b736-92a97ccdafed","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:45.191Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"69811076-6ccf-42fb-b736-92a97ccdafed","filter labels":["test-ns-ltx56",{"ray.io/cluster":"stop-9cztm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:00:45.191Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"69811076-6ccf-42fb-b736-92a97ccdafed"}
{"level":"error","ts":"2026-09-18T10:00:45.191Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"69811076-6ccf-42fb-b736-92a97ccdafed","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-9cztm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:45.508Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-ltx56"},"reconcileID":"9e8c1039-7cd6-4b66-95e9-c6b5a111434f"}
{"level":"info","ts":"2026-09-18T10:00:46.472Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae"}
{"level":"info","ts":"2026-09-18T10:00:46.472Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae"}
{"level":"info","ts":"2026-09-18T10:00:46.472Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae"}
{"level":"info","ts":"2026-09-18T10:00:46.472Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:46.472Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae"}
{"level":"info","ts":"2026-09-18T10:00:46.472Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:00:46.472Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae"}
{"level":"info","ts":"2026-09-18T10:00:46.472Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","mode":"IntegratedOAuth","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:46.472Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","cluster":"stop-9cztm","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:00:46.472Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","cluster":"stop-9cztm","authType":"IntegratedOAuth","serviceAccount":"stop-9cztm-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T10:00:46.472Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"stop-9cztm","ray.io/group":"headgroup","ray.io/identifier":"stop-9cztm-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:00:46.472Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T10:00:46.472Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:00:46.472Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:00:46.472Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","enabled":true}
{"level":"info","ts":"2026-09-18T10:00:46.490Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","name":"stop-9cztm-head-jvvdj"}
{"level":"info","ts":"2026-09-18T10:00:46.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:00:46.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:46.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:00:46.490Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:00:46.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:46.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:00:46.490Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae"}
{"level":"info","ts":"2026-09-18T10:00:46.490Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","nodeType":"worker","rayStartParams":{"address":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:00:46.490Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","rayStartCmd":"ray start  --address=stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:00:46.490Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:00:46.490Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","enabled":true}
{"level":"info","ts":"2026-09-18T10:00:46.505Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","name":"stop-9cztm-small-group-worker-w9ml2"}
{"level":"info","ts":"2026-09-18T10:00:46.505Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:46.505Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:46.505Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"5badb86d-2b10-4dc5-9c62-110a0b4dcdae","name":"stop-9cztm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242"}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242"}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242"}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e3d14d8a-ce7b-498a-bd26-d10c5044f24d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e3d14d8a-ce7b-498a-bd26-d10c5044f24d","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e3d14d8a-ce7b-498a-bd26-d10c5044f24d","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e3d14d8a-ce7b-498a-bd26-d10c5044f24d"}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e3d14d8a-ce7b-498a-bd26-d10c5044f24d","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e3d14d8a-ce7b-498a-bd26-d10c5044f24d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e3d14d8a-ce7b-498a-bd26-d10c5044f24d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:00:46.521Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"a812f306-4ffe-4e24-b76e-a5154be0fc14","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:46.521Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"a812f306-4ffe-4e24-b76e-a5154be0fc14","rayCluster":"stop-9cztm","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:00:46.521Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"4a74a7e7-6300-4069-9104-6f09605d569b","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:46.520Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242","name":"stop-9cztm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:00:46.566Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"4a74a7e7-6300-4069-9104-6f09605d569b","name":"stop-9cztm-head"}
{"level":"info","ts":"2026-09-18T10:00:46.567Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"25c80916-f0a6-49cb-a2dd-fced09f15bf4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:46.567Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"25c80916-f0a6-49cb-a2dd-fced09f15bf4","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:46.567Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"25c80916-f0a6-49cb-a2dd-fced09f15bf4","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:46.567Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"25c80916-f0a6-49cb-a2dd-fced09f15bf4"}
{"level":"info","ts":"2026-09-18T10:00:46.567Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"25c80916-f0a6-49cb-a2dd-fced09f15bf4","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:46.567Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"25c80916-f0a6-49cb-a2dd-fced09f15bf4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:46.574Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242","name":"stop-9cztm","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-9cztm\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"stop-9cztm"}}
{"level":"info","ts":"2026-09-18T10:00:46.574Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242"}
{"level":"error","ts":"2026-09-18T10:00:46.574Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d1fa8f10-1a66-4d3a-953c-65765ec07242","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-9cztm\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:46.574Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c8fe45e0-8c18-4569-8c91-c0ce42b081b6"}
{"level":"info","ts":"2026-09-18T10:00:46.574Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c8fe45e0-8c18-4569-8c91-c0ce42b081b6"}
{"level":"info","ts":"2026-09-18T10:00:46.574Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c8fe45e0-8c18-4569-8c91-c0ce42b081b6"}
{"level":"info","ts":"2026-09-18T10:00:46.574Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c8fe45e0-8c18-4569-8c91-c0ce42b081b6","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:46.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c8fe45e0-8c18-4569-8c91-c0ce42b081b6","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:00:46.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c8fe45e0-8c18-4569-8c91-c0ce42b081b6","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:46.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c8fe45e0-8c18-4569-8c91-c0ce42b081b6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:00:46.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c8fe45e0-8c18-4569-8c91-c0ce42b081b6","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:46.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c8fe45e0-8c18-4569-8c91-c0ce42b081b6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:46.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c8fe45e0-8c18-4569-8c91-c0ce42b081b6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:00:46.574Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c8fe45e0-8c18-4569-8c91-c0ce42b081b6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:00:46.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c8fe45e0-8c18-4569-8c91-c0ce42b081b6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:46.575Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c8fe45e0-8c18-4569-8c91-c0ce42b081b6","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:46.575Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c8fe45e0-8c18-4569-8c91-c0ce42b081b6","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:46.575Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c8fe45e0-8c18-4569-8c91-c0ce42b081b6","name":"stop-9cztm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:00:46.578Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"4a74a7e7-6300-4069-9104-6f09605d569b","name":"stop-9cztm-workers"}
{"level":"info","ts":"2026-09-18T10:00:46.578Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"4a74a7e7-6300-4069-9104-6f09605d569b","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96"}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96"}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96"}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"cc067428-eb33-47e1-8208-dc1fc4deb924","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96","name":"stop-9cztm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"02a24ec7-c913-4207-8b9d-47e2669f0802","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:46.587Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"02a24ec7-c913-4207-8b9d-47e2669f0802","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:46.588Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"02a24ec7-c913-4207-8b9d-47e2669f0802","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:46.588Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"02a24ec7-c913-4207-8b9d-47e2669f0802"}
{"level":"info","ts":"2026-09-18T10:00:46.588Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"02a24ec7-c913-4207-8b9d-47e2669f0802","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:46.588Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"02a24ec7-c913-4207-8b9d-47e2669f0802","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:46.588Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"02a24ec7-c913-4207-8b9d-47e2669f0802","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:00:46.588Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"6cfc6a4f-8b1c-497d-8502-c8614216b29d","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:46.588Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"6cfc6a4f-8b1c-497d-8502-c8614216b29d","rayCluster":"stop-9cztm","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:00:46.671Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"cc067428-eb33-47e1-8208-dc1fc4deb924","name":"stop-9cztm-head"}
{"level":"info","ts":"2026-09-18T10:00:46.674Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96","name":"stop-9cztm","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-9cztm\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"stop-9cztm"}}
{"level":"info","ts":"2026-09-18T10:00:46.674Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96"}
{"level":"error","ts":"2026-09-18T10:00:46.674Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1e19f164-3cca-4ade-926e-3fcdff900a96","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-9cztm\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:00:46.679Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"56e6cb3a-3867-47fb-8040-d52f8fc35853"}
{"level":"info","ts":"2026-09-18T10:00:46.679Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"56e6cb3a-3867-47fb-8040-d52f8fc35853"}
{"level":"info","ts":"2026-09-18T10:00:46.679Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"56e6cb3a-3867-47fb-8040-d52f8fc35853"}
{"level":"info","ts":"2026-09-18T10:00:46.679Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"56e6cb3a-3867-47fb-8040-d52f8fc35853","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:46.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"56e6cb3a-3867-47fb-8040-d52f8fc35853","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:00:46.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"56e6cb3a-3867-47fb-8040-d52f8fc35853","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:46.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"56e6cb3a-3867-47fb-8040-d52f8fc35853","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:00:46.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"56e6cb3a-3867-47fb-8040-d52f8fc35853","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:46.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"56e6cb3a-3867-47fb-8040-d52f8fc35853","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:46.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"56e6cb3a-3867-47fb-8040-d52f8fc35853","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:00:46.679Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"56e6cb3a-3867-47fb-8040-d52f8fc35853","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:00:46.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"56e6cb3a-3867-47fb-8040-d52f8fc35853","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:46.679Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"56e6cb3a-3867-47fb-8040-d52f8fc35853","podName":"stop-9cztm-small-group-worker-w9ml2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:46.679Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"56e6cb3a-3867-47fb-8040-d52f8fc35853","podName":"stop-9cztm-small-group-worker-w9ml2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:46.679Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"56e6cb3a-3867-47fb-8040-d52f8fc35853","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:00:46.680Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"56e6cb3a-3867-47fb-8040-d52f8fc35853","seconds":300}
{"level":"info","ts":"2026-09-18T10:00:46.681Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5c3bb5b6-4dce-4a40-8671-1aed054ed867","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:46.681Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5c3bb5b6-4dce-4a40-8671-1aed054ed867","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:46.681Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5c3bb5b6-4dce-4a40-8671-1aed054ed867","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:46.681Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5c3bb5b6-4dce-4a40-8671-1aed054ed867"}
{"level":"info","ts":"2026-09-18T10:00:46.681Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5c3bb5b6-4dce-4a40-8671-1aed054ed867","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:46.681Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5c3bb5b6-4dce-4a40-8671-1aed054ed867","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:46.689Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"cc067428-eb33-47e1-8208-dc1fc4deb924","name":"stop-9cztm-workers"}
{"level":"info","ts":"2026-09-18T10:00:46.689Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"cc067428-eb33-47e1-8208-dc1fc4deb924","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:46.728Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fcd4e56a-4286-426e-aae1-7a65ad116217","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:46.728Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fcd4e56a-4286-426e-aae1-7a65ad116217","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:46.728Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fcd4e56a-4286-426e-aae1-7a65ad116217","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:46.728Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fcd4e56a-4286-426e-aae1-7a65ad116217"}
{"level":"info","ts":"2026-09-18T10:00:46.728Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fcd4e56a-4286-426e-aae1-7a65ad116217","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:46.728Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fcd4e56a-4286-426e-aae1-7a65ad116217","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:47.099Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"58c29b58-0779-4092-856c-ee317555f36f"}
{"level":"info","ts":"2026-09-18T10:00:47.099Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"58c29b58-0779-4092-856c-ee317555f36f"}
{"level":"info","ts":"2026-09-18T10:00:47.099Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"58c29b58-0779-4092-856c-ee317555f36f"}
{"level":"info","ts":"2026-09-18T10:00:47.099Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"58c29b58-0779-4092-856c-ee317555f36f","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:47.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"58c29b58-0779-4092-856c-ee317555f36f","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:00:47.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"58c29b58-0779-4092-856c-ee317555f36f","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:47.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"58c29b58-0779-4092-856c-ee317555f36f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:00:47.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"58c29b58-0779-4092-856c-ee317555f36f","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:47.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"58c29b58-0779-4092-856c-ee317555f36f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:47.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"58c29b58-0779-4092-856c-ee317555f36f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:00:47.099Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"58c29b58-0779-4092-856c-ee317555f36f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:00:47.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"58c29b58-0779-4092-856c-ee317555f36f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:47.099Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"58c29b58-0779-4092-856c-ee317555f36f","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:47.099Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"58c29b58-0779-4092-856c-ee317555f36f","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:47.099Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"58c29b58-0779-4092-856c-ee317555f36f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:00:47.099Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"58c29b58-0779-4092-856c-ee317555f36f","seconds":300}
{"level":"info","ts":"2026-09-18T10:00:47.122Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3d45bc5-c82c-45e5-95b4-bebbc6dc2bb0"}
{"level":"info","ts":"2026-09-18T10:00:47.122Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3d45bc5-c82c-45e5-95b4-bebbc6dc2bb0"}
{"level":"info","ts":"2026-09-18T10:00:47.122Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3d45bc5-c82c-45e5-95b4-bebbc6dc2bb0"}
{"level":"info","ts":"2026-09-18T10:00:47.122Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3d45bc5-c82c-45e5-95b4-bebbc6dc2bb0","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:47.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3d45bc5-c82c-45e5-95b4-bebbc6dc2bb0","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:00:47.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3d45bc5-c82c-45e5-95b4-bebbc6dc2bb0","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:47.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3d45bc5-c82c-45e5-95b4-bebbc6dc2bb0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:00:47.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3d45bc5-c82c-45e5-95b4-bebbc6dc2bb0","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:47.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3d45bc5-c82c-45e5-95b4-bebbc6dc2bb0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:47.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3d45bc5-c82c-45e5-95b4-bebbc6dc2bb0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:00:47.123Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3d45bc5-c82c-45e5-95b4-bebbc6dc2bb0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:00:47.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3d45bc5-c82c-45e5-95b4-bebbc6dc2bb0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:47.123Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3d45bc5-c82c-45e5-95b4-bebbc6dc2bb0","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:47.123Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3d45bc5-c82c-45e5-95b4-bebbc6dc2bb0","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:47.123Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3d45bc5-c82c-45e5-95b4-bebbc6dc2bb0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:00:47.123Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"c3d45bc5-c82c-45e5-95b4-bebbc6dc2bb0","seconds":300}
{"level":"info","ts":"2026-09-18T10:00:47.501Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"65fd05c2-968c-42b8-9486-841aba5b591d"}
{"level":"info","ts":"2026-09-18T10:00:47.501Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"65fd05c2-968c-42b8-9486-841aba5b591d"}
{"level":"info","ts":"2026-09-18T10:00:47.501Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"65fd05c2-968c-42b8-9486-841aba5b591d"}
{"level":"info","ts":"2026-09-18T10:00:47.501Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"65fd05c2-968c-42b8-9486-841aba5b591d","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:47.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"65fd05c2-968c-42b8-9486-841aba5b591d","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:00:47.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"65fd05c2-968c-42b8-9486-841aba5b591d","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:47.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"65fd05c2-968c-42b8-9486-841aba5b591d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:00:47.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"65fd05c2-968c-42b8-9486-841aba5b591d","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:47.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"65fd05c2-968c-42b8-9486-841aba5b591d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:47.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"65fd05c2-968c-42b8-9486-841aba5b591d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:00:47.501Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"65fd05c2-968c-42b8-9486-841aba5b591d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:00:47.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"65fd05c2-968c-42b8-9486-841aba5b591d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:47.501Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"65fd05c2-968c-42b8-9486-841aba5b591d","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:47.501Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"65fd05c2-968c-42b8-9486-841aba5b591d","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:47.502Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"65fd05c2-968c-42b8-9486-841aba5b591d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:00:47.502Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"65fd05c2-968c-42b8-9486-841aba5b591d","seconds":300}
{"level":"info","ts":"2026-09-18T10:00:48.020Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9a882708-5642-4877-ba7b-8f5e2685f3f4"}
{"level":"info","ts":"2026-09-18T10:00:48.020Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9a882708-5642-4877-ba7b-8f5e2685f3f4"}
{"level":"info","ts":"2026-09-18T10:00:48.020Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9a882708-5642-4877-ba7b-8f5e2685f3f4"}
{"level":"info","ts":"2026-09-18T10:00:48.020Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9a882708-5642-4877-ba7b-8f5e2685f3f4","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:48.020Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9a882708-5642-4877-ba7b-8f5e2685f3f4","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:00:48.020Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9a882708-5642-4877-ba7b-8f5e2685f3f4","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:48.020Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9a882708-5642-4877-ba7b-8f5e2685f3f4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:00:48.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9a882708-5642-4877-ba7b-8f5e2685f3f4","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:48.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9a882708-5642-4877-ba7b-8f5e2685f3f4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:48.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9a882708-5642-4877-ba7b-8f5e2685f3f4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:00:48.021Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9a882708-5642-4877-ba7b-8f5e2685f3f4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:00:48.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9a882708-5642-4877-ba7b-8f5e2685f3f4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:48.021Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9a882708-5642-4877-ba7b-8f5e2685f3f4","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:48.021Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9a882708-5642-4877-ba7b-8f5e2685f3f4","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:48.021Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9a882708-5642-4877-ba7b-8f5e2685f3f4","name":"stop-9cztm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:00:48.066Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"93d32cb2-faaa-4d5b-8b8c-436f1d647dbb","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:48.066Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"93d32cb2-faaa-4d5b-8b8c-436f1d647dbb","pod":"stop-9cztm-head-jvvdj","ip":"10.133.0.41"}
{"level":"info","ts":"2026-09-18T10:00:48.066Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"93d32cb2-faaa-4d5b-8b8c-436f1d647dbb","pod":"stop-9cztm-small-group-worker-w9ml2","ip":"10.132.0.45"}
{"level":"info","ts":"2026-09-18T10:00:48.067Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8a093010-fb73-4185-8697-def436aecd68","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:48.067Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8a093010-fb73-4185-8697-def436aecd68","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:48.067Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8a093010-fb73-4185-8697-def436aecd68","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:48.067Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8a093010-fb73-4185-8697-def436aecd68"}
{"level":"info","ts":"2026-09-18T10:00:48.067Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8a093010-fb73-4185-8697-def436aecd68","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:48.067Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8a093010-fb73-4185-8697-def436aecd68","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:48.067Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8a093010-fb73-4185-8697-def436aecd68","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:00:48.067Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"780fd7b3-e861-4ea2-9e43-e03511e12bdd","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:48.080Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"93d32cb2-faaa-4d5b-8b8c-436f1d647dbb","certificate":"ray-head-cert-stop-9cztm","dnsNames":["stop-9cztm-head-svc","localhost","stop-9cztm-head-svc.test-ns-ltx56.svc","stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.41","10.132.0.45","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:00:48.083Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"780fd7b3-e861-4ea2-9e43-e03511e12bdd","name":"stop-9cztm-head"}
{"level":"info","ts":"2026-09-18T10:00:48.085Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d222e472-fe16-468e-904c-79a69e502ece","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:48.085Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d222e472-fe16-468e-904c-79a69e502ece","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:48.085Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d222e472-fe16-468e-904c-79a69e502ece","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:48.085Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d222e472-fe16-468e-904c-79a69e502ece"}
{"level":"info","ts":"2026-09-18T10:00:48.085Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d222e472-fe16-468e-904c-79a69e502ece","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:48.085Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d222e472-fe16-468e-904c-79a69e502ece","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:48.094Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"93d32cb2-faaa-4d5b-8b8c-436f1d647dbb","certificate":"ray-worker-cert-stop-9cztm","dnsNames":["stop-9cztm-worker-svc","localhost","stop-9cztm-small-group","stop-9cztm-worker-svc.test-ns-ltx56.svc","stop-9cztm-worker-svc.test-ns-ltx56.svc.cluster.local","*.stop-9cztm-worker-svc.test-ns-ltx56.svc","*.stop-9cztm-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.41","10.132.0.45","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:00:48.094Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"93d32cb2-faaa-4d5b-8b8c-436f1d647dbb","rayCluster":"stop-9cztm","podIPs":["10.133.0.41","10.132.0.45"]}
{"level":"info","ts":"2026-09-18T10:00:48.096Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"780fd7b3-e861-4ea2-9e43-e03511e12bdd","name":"stop-9cztm-workers"}
{"level":"info","ts":"2026-09-18T10:00:48.096Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"780fd7b3-e861-4ea2-9e43-e03511e12bdd","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:49.728Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e3d218d5-57f7-4ce8-88be-5a04fced1cfd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:49.728Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e3d218d5-57f7-4ce8-88be-5a04fced1cfd","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:49.729Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e3d218d5-57f7-4ce8-88be-5a04fced1cfd","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:49.729Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e3d218d5-57f7-4ce8-88be-5a04fced1cfd"}
{"level":"info","ts":"2026-09-18T10:00:49.729Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e3d218d5-57f7-4ce8-88be-5a04fced1cfd","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:49.729Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e3d218d5-57f7-4ce8-88be-5a04fced1cfd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:50.067Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"71aa2fb2-faa0-44de-a89d-4c62ea98974b"}
{"level":"info","ts":"2026-09-18T10:00:50.067Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"71aa2fb2-faa0-44de-a89d-4c62ea98974b"}
{"level":"info","ts":"2026-09-18T10:00:50.067Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"71aa2fb2-faa0-44de-a89d-4c62ea98974b"}
{"level":"info","ts":"2026-09-18T10:00:50.067Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"71aa2fb2-faa0-44de-a89d-4c62ea98974b","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:50.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"71aa2fb2-faa0-44de-a89d-4c62ea98974b","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:00:50.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"71aa2fb2-faa0-44de-a89d-4c62ea98974b","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:50.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"71aa2fb2-faa0-44de-a89d-4c62ea98974b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:00:50.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"71aa2fb2-faa0-44de-a89d-4c62ea98974b","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:50.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"71aa2fb2-faa0-44de-a89d-4c62ea98974b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:50.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"71aa2fb2-faa0-44de-a89d-4c62ea98974b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:00:50.068Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"71aa2fb2-faa0-44de-a89d-4c62ea98974b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:00:50.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"71aa2fb2-faa0-44de-a89d-4c62ea98974b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:50.068Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"71aa2fb2-faa0-44de-a89d-4c62ea98974b","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:50.068Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"71aa2fb2-faa0-44de-a89d-4c62ea98974b","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:50.068Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"71aa2fb2-faa0-44de-a89d-4c62ea98974b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:00:50.068Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"71aa2fb2-faa0-44de-a89d-4c62ea98974b","seconds":300}
{"level":"info","ts":"2026-09-18T10:00:52.730Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"14bc602e-8c09-41e2-a641-ebb7a12c1eec","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:52.730Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"14bc602e-8c09-41e2-a641-ebb7a12c1eec","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:52.730Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"14bc602e-8c09-41e2-a641-ebb7a12c1eec","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:52.730Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"14bc602e-8c09-41e2-a641-ebb7a12c1eec"}
{"level":"info","ts":"2026-09-18T10:00:52.730Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"14bc602e-8c09-41e2-a641-ebb7a12c1eec","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:52.730Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"14bc602e-8c09-41e2-a641-ebb7a12c1eec","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:53.021Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9e6d567d-b75e-4929-9689-d08679ac016f"}
{"level":"info","ts":"2026-09-18T10:00:53.021Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9e6d567d-b75e-4929-9689-d08679ac016f"}
{"level":"info","ts":"2026-09-18T10:00:53.021Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9e6d567d-b75e-4929-9689-d08679ac016f"}
{"level":"info","ts":"2026-09-18T10:00:53.021Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9e6d567d-b75e-4929-9689-d08679ac016f","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:53.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9e6d567d-b75e-4929-9689-d08679ac016f","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:00:53.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9e6d567d-b75e-4929-9689-d08679ac016f","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:53.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9e6d567d-b75e-4929-9689-d08679ac016f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:00:53.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9e6d567d-b75e-4929-9689-d08679ac016f","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:53.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9e6d567d-b75e-4929-9689-d08679ac016f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:53.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9e6d567d-b75e-4929-9689-d08679ac016f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:00:53.022Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9e6d567d-b75e-4929-9689-d08679ac016f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:00:53.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9e6d567d-b75e-4929-9689-d08679ac016f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:53.022Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9e6d567d-b75e-4929-9689-d08679ac016f","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:53.022Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9e6d567d-b75e-4929-9689-d08679ac016f","podName":"stop-9cztm-head-jvvdj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:00:53.022Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9e6d567d-b75e-4929-9689-d08679ac016f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:00:53.022Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"9e6d567d-b75e-4929-9689-d08679ac016f","seconds":300}
{"level":"info","ts":"2026-09-18T10:00:54.041Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"93938741-5d9d-45f3-9b6a-624fae97ea3f"}
{"level":"info","ts":"2026-09-18T10:00:54.041Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"93938741-5d9d-45f3-9b6a-624fae97ea3f"}
{"level":"info","ts":"2026-09-18T10:00:54.041Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"93938741-5d9d-45f3-9b6a-624fae97ea3f"}
{"level":"info","ts":"2026-09-18T10:00:54.041Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"93938741-5d9d-45f3-9b6a-624fae97ea3f","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:54.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"93938741-5d9d-45f3-9b6a-624fae97ea3f","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:00:54.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"93938741-5d9d-45f3-9b6a-624fae97ea3f","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:54.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"93938741-5d9d-45f3-9b6a-624fae97ea3f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:00:54.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"93938741-5d9d-45f3-9b6a-624fae97ea3f","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:54.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"93938741-5d9d-45f3-9b6a-624fae97ea3f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:54.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"93938741-5d9d-45f3-9b6a-624fae97ea3f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:00:54.041Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"93938741-5d9d-45f3-9b6a-624fae97ea3f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:00:54.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"93938741-5d9d-45f3-9b6a-624fae97ea3f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:54.041Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"93938741-5d9d-45f3-9b6a-624fae97ea3f","podName":"stop-9cztm-head-jvvdj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:00:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:00:54.041Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"93938741-5d9d-45f3-9b6a-624fae97ea3f","podName":"stop-9cztm-head-jvvdj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:00:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:00:54.042Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"93938741-5d9d-45f3-9b6a-624fae97ea3f","name":"stop-9cztm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:00:54.056Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"bc186d44-5814-4a7d-8603-65ea77ed4099","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:54.056Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"80c8c110-c3eb-4182-b8ee-f2f4138a3555","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:54.056Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"80c8c110-c3eb-4182-b8ee-f2f4138a3555","pod":"stop-9cztm-head-jvvdj","ip":"10.133.0.41"}
{"level":"info","ts":"2026-09-18T10:00:54.056Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"80c8c110-c3eb-4182-b8ee-f2f4138a3555","pod":"stop-9cztm-small-group-worker-w9ml2","ip":"10.132.0.45"}
{"level":"info","ts":"2026-09-18T10:00:54.056Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"80c8c110-c3eb-4182-b8ee-f2f4138a3555","rayCluster":"stop-9cztm","podIPs":["10.133.0.41","10.132.0.45"]}
{"level":"info","ts":"2026-09-18T10:00:54.056Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"52e2cdab-2fd5-40d8-82e7-911acba2bf95","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:54.056Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"52e2cdab-2fd5-40d8-82e7-911acba2bf95","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:54.056Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"52e2cdab-2fd5-40d8-82e7-911acba2bf95","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:54.056Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"52e2cdab-2fd5-40d8-82e7-911acba2bf95"}
{"level":"info","ts":"2026-09-18T10:00:54.056Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"52e2cdab-2fd5-40d8-82e7-911acba2bf95","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:54.056Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"52e2cdab-2fd5-40d8-82e7-911acba2bf95","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:54.057Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"52e2cdab-2fd5-40d8-82e7-911acba2bf95","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:00:54.067Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"bc186d44-5814-4a7d-8603-65ea77ed4099","name":"stop-9cztm-head"}
{"level":"info","ts":"2026-09-18T10:00:54.074Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"453924ab-dd61-4c70-b84b-53b1e069b543","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:54.074Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"453924ab-dd61-4c70-b84b-53b1e069b543","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:54.074Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"453924ab-dd61-4c70-b84b-53b1e069b543","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:54.074Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"453924ab-dd61-4c70-b84b-53b1e069b543"}
{"level":"info","ts":"2026-09-18T10:00:54.074Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"453924ab-dd61-4c70-b84b-53b1e069b543","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:54.074Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"453924ab-dd61-4c70-b84b-53b1e069b543","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:54.096Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"bc186d44-5814-4a7d-8603-65ea77ed4099","name":"stop-9cztm-workers"}
{"level":"info","ts":"2026-09-18T10:00:54.096Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"bc186d44-5814-4a7d-8603-65ea77ed4099","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:55.731Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bc034ae8-de5b-4c32-8553-3ef9a4794da0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:55.731Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bc034ae8-de5b-4c32-8553-3ef9a4794da0","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:55.731Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bc034ae8-de5b-4c32-8553-3ef9a4794da0","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:55.731Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bc034ae8-de5b-4c32-8553-3ef9a4794da0"}
{"level":"info","ts":"2026-09-18T10:00:55.731Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bc034ae8-de5b-4c32-8553-3ef9a4794da0","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:55.731Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bc034ae8-de5b-4c32-8553-3ef9a4794da0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:00:56.057Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"820f4003-7e77-493b-8079-667a5e6813c7"}
{"level":"info","ts":"2026-09-18T10:00:56.057Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"820f4003-7e77-493b-8079-667a5e6813c7"}
{"level":"info","ts":"2026-09-18T10:00:56.057Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"820f4003-7e77-493b-8079-667a5e6813c7"}
{"level":"info","ts":"2026-09-18T10:00:56.057Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"820f4003-7e77-493b-8079-667a5e6813c7","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:00:56.057Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"820f4003-7e77-493b-8079-667a5e6813c7","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:00:56.057Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"820f4003-7e77-493b-8079-667a5e6813c7","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:56.057Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"820f4003-7e77-493b-8079-667a5e6813c7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:00:56.057Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"820f4003-7e77-493b-8079-667a5e6813c7","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:00:56.057Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"820f4003-7e77-493b-8079-667a5e6813c7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:56.057Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"820f4003-7e77-493b-8079-667a5e6813c7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:00:56.057Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"820f4003-7e77-493b-8079-667a5e6813c7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:00:56.057Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"820f4003-7e77-493b-8079-667a5e6813c7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:00:56.057Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"820f4003-7e77-493b-8079-667a5e6813c7","podName":"stop-9cztm-head-jvvdj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:00:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:00:56.057Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"820f4003-7e77-493b-8079-667a5e6813c7","podName":"stop-9cztm-head-jvvdj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:00:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:00:56.057Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"820f4003-7e77-493b-8079-667a5e6813c7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:00:56.057Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"820f4003-7e77-493b-8079-667a5e6813c7","seconds":300}
{"level":"info","ts":"2026-09-18T10:00:58.732Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f34ffc03-7ce9-4c17-97b2-ed33b1e55a96","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:00:58.732Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f34ffc03-7ce9-4c17-97b2-ed33b1e55a96","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:00:58.732Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f34ffc03-7ce9-4c17-97b2-ed33b1e55a96","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:00:58.732Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f34ffc03-7ce9-4c17-97b2-ed33b1e55a96"}
{"level":"info","ts":"2026-09-18T10:00:58.732Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f34ffc03-7ce9-4c17-97b2-ed33b1e55a96","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:00:58.732Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f34ffc03-7ce9-4c17-97b2-ed33b1e55a96","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:01.733Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5b0d284d-16b9-4f72-a7d9-8b3b556ba6e9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:01.733Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5b0d284d-16b9-4f72-a7d9-8b3b556ba6e9","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:01.733Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5b0d284d-16b9-4f72-a7d9-8b3b556ba6e9","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:01.733Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5b0d284d-16b9-4f72-a7d9-8b3b556ba6e9"}
{"level":"info","ts":"2026-09-18T10:01:01.733Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5b0d284d-16b9-4f72-a7d9-8b3b556ba6e9","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:01.733Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5b0d284d-16b9-4f72-a7d9-8b3b556ba6e9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:04.734Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"edcab569-7b74-42e9-bf44-a5d510d702c5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:04.734Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"edcab569-7b74-42e9-bf44-a5d510d702c5","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:04.734Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"edcab569-7b74-42e9-bf44-a5d510d702c5","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:04.734Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"edcab569-7b74-42e9-bf44-a5d510d702c5"}
{"level":"info","ts":"2026-09-18T10:01:04.734Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"edcab569-7b74-42e9-bf44-a5d510d702c5","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:04.734Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"edcab569-7b74-42e9-bf44-a5d510d702c5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:05.055Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a1f087b7-8f55-4039-bb4a-6b34592c11d0"}
{"level":"info","ts":"2026-09-18T10:01:05.055Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a1f087b7-8f55-4039-bb4a-6b34592c11d0"}
{"level":"info","ts":"2026-09-18T10:01:05.056Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a1f087b7-8f55-4039-bb4a-6b34592c11d0"}
{"level":"info","ts":"2026-09-18T10:01:05.056Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a1f087b7-8f55-4039-bb4a-6b34592c11d0","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:01:05.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a1f087b7-8f55-4039-bb4a-6b34592c11d0","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:01:05.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a1f087b7-8f55-4039-bb4a-6b34592c11d0","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:01:05.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a1f087b7-8f55-4039-bb4a-6b34592c11d0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:01:05.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a1f087b7-8f55-4039-bb4a-6b34592c11d0","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:01:05.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a1f087b7-8f55-4039-bb4a-6b34592c11d0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:01:05.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a1f087b7-8f55-4039-bb4a-6b34592c11d0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:01:05.056Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a1f087b7-8f55-4039-bb4a-6b34592c11d0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:01:05.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a1f087b7-8f55-4039-bb4a-6b34592c11d0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:01:05.056Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a1f087b7-8f55-4039-bb4a-6b34592c11d0","podName":"stop-9cztm-small-group-worker-w9ml2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:01:05.056Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a1f087b7-8f55-4039-bb4a-6b34592c11d0","podName":"stop-9cztm-small-group-worker-w9ml2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:01:05.056Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a1f087b7-8f55-4039-bb4a-6b34592c11d0","name":"stop-9cztm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:01:05.070Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"f5d5458d-d930-47f7-8ed5-f60376f8b8b9","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:05.070Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"7799ec9f-cf23-4b9e-ace7-7667eb29736d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:05.070Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"7799ec9f-cf23-4b9e-ace7-7667eb29736d","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:05.070Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"7799ec9f-cf23-4b9e-ace7-7667eb29736d","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:05.070Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"f5d5458d-d930-47f7-8ed5-f60376f8b8b9","pod":"stop-9cztm-head-jvvdj","ip":"10.133.0.41"}
{"level":"info","ts":"2026-09-18T10:01:05.070Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"f5d5458d-d930-47f7-8ed5-f60376f8b8b9","pod":"stop-9cztm-small-group-worker-w9ml2","ip":"10.132.0.45"}
{"level":"info","ts":"2026-09-18T10:01:05.070Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"7799ec9f-cf23-4b9e-ace7-7667eb29736d"}
{"level":"info","ts":"2026-09-18T10:01:05.070Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"f5d5458d-d930-47f7-8ed5-f60376f8b8b9","rayCluster":"stop-9cztm","podIPs":["10.133.0.41","10.132.0.45"]}
{"level":"info","ts":"2026-09-18T10:01:05.070Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"7799ec9f-cf23-4b9e-ace7-7667eb29736d","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:05.070Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"7799ec9f-cf23-4b9e-ace7-7667eb29736d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:00:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:05.070Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"7799ec9f-cf23-4b9e-ace7-7667eb29736d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:01:05.071Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"b9ee5d0a-b77a-44cf-ad41-a1f863dcafb6","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:05.084Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"9a15fd15-41ac-44ee-a0dc-60d79f7525ac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:05.084Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"9a15fd15-41ac-44ee-a0dc-60d79f7525ac","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:05.084Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"9a15fd15-41ac-44ee-a0dc-60d79f7525ac","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:05.084Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"9a15fd15-41ac-44ee-a0dc-60d79f7525ac"}
{"level":"info","ts":"2026-09-18T10:01:05.084Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"9a15fd15-41ac-44ee-a0dc-60d79f7525ac","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:05.084Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"9a15fd15-41ac-44ee-a0dc-60d79f7525ac","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:05.089Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"b9ee5d0a-b77a-44cf-ad41-a1f863dcafb6","name":"stop-9cztm-head"}
{"level":"info","ts":"2026-09-18T10:01:05.101Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"b9ee5d0a-b77a-44cf-ad41-a1f863dcafb6","name":"stop-9cztm-workers"}
{"level":"info","ts":"2026-09-18T10:01:05.101Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"b9ee5d0a-b77a-44cf-ad41-a1f863dcafb6","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:07.071Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a38a91bd-82ef-4dbc-b208-8499313d5a1b"}
{"level":"info","ts":"2026-09-18T10:01:07.071Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a38a91bd-82ef-4dbc-b208-8499313d5a1b"}
{"level":"info","ts":"2026-09-18T10:01:07.071Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a38a91bd-82ef-4dbc-b208-8499313d5a1b"}
{"level":"info","ts":"2026-09-18T10:01:07.071Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a38a91bd-82ef-4dbc-b208-8499313d5a1b","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:01:07.071Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a38a91bd-82ef-4dbc-b208-8499313d5a1b","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:01:07.071Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a38a91bd-82ef-4dbc-b208-8499313d5a1b","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:01:07.071Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a38a91bd-82ef-4dbc-b208-8499313d5a1b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:01:07.071Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a38a91bd-82ef-4dbc-b208-8499313d5a1b","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:01:07.071Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a38a91bd-82ef-4dbc-b208-8499313d5a1b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:01:07.071Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a38a91bd-82ef-4dbc-b208-8499313d5a1b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:01:07.071Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a38a91bd-82ef-4dbc-b208-8499313d5a1b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:01:07.071Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a38a91bd-82ef-4dbc-b208-8499313d5a1b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:01:07.071Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a38a91bd-82ef-4dbc-b208-8499313d5a1b","podName":"stop-9cztm-small-group-worker-w9ml2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:01:07.071Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a38a91bd-82ef-4dbc-b208-8499313d5a1b","podName":"stop-9cztm-small-group-worker-w9ml2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:01:07.072Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a38a91bd-82ef-4dbc-b208-8499313d5a1b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:01:07.072Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"a38a91bd-82ef-4dbc-b208-8499313d5a1b","seconds":300}
{"level":"info","ts":"2026-09-18T10:01:07.735Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0271b92c-f48e-4dd5-8ce7-a36a9b8b8a38","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:07.735Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0271b92c-f48e-4dd5-8ce7-a36a9b8b8a38","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:07.735Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0271b92c-f48e-4dd5-8ce7-a36a9b8b8a38","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:07.735Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0271b92c-f48e-4dd5-8ce7-a36a9b8b8a38"}
{"level":"info","ts":"2026-09-18T10:01:07.735Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0271b92c-f48e-4dd5-8ce7-a36a9b8b8a38","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:07.735Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0271b92c-f48e-4dd5-8ce7-a36a9b8b8a38","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:10.736Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"1432ef12-9bcd-47ff-8474-e3a0a59f1cfa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:10.736Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"1432ef12-9bcd-47ff-8474-e3a0a59f1cfa","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:10.736Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"1432ef12-9bcd-47ff-8474-e3a0a59f1cfa","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:10.736Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"1432ef12-9bcd-47ff-8474-e3a0a59f1cfa"}
{"level":"info","ts":"2026-09-18T10:01:10.736Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"1432ef12-9bcd-47ff-8474-e3a0a59f1cfa","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:10.736Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"1432ef12-9bcd-47ff-8474-e3a0a59f1cfa","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:13.736Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"013f6940-db17-4e79-b79b-f51283a917b6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:13.736Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"013f6940-db17-4e79-b79b-f51283a917b6","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:13.736Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"013f6940-db17-4e79-b79b-f51283a917b6","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:13.736Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"013f6940-db17-4e79-b79b-f51283a917b6"}
{"level":"info","ts":"2026-09-18T10:01:13.736Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"013f6940-db17-4e79-b79b-f51283a917b6","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:13.736Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"013f6940-db17-4e79-b79b-f51283a917b6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:14.108Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"6a46b6d2-cf8b-49fe-bf3c-48b2ef065fcd","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:14.108Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"6a46b6d2-cf8b-49fe-bf3c-48b2ef065fcd","pod":"stop-9cztm-head-jvvdj","ip":"10.133.0.41"}
{"level":"info","ts":"2026-09-18T10:01:14.108Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"6a46b6d2-cf8b-49fe-bf3c-48b2ef065fcd","pod":"stop-9cztm-small-group-worker-w9ml2","ip":"10.132.0.45"}
{"level":"info","ts":"2026-09-18T10:01:14.108Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"6a46b6d2-cf8b-49fe-bf3c-48b2ef065fcd","rayCluster":"stop-9cztm","podIPs":["10.133.0.41","10.132.0.45"]}
{"level":"info","ts":"2026-09-18T10:01:16.737Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"30af3134-a980-4c6e-aa99-258736d6f8a2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:16.737Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"30af3134-a980-4c6e-aa99-258736d6f8a2","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:16.737Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"30af3134-a980-4c6e-aa99-258736d6f8a2","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:16.737Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"30af3134-a980-4c6e-aa99-258736d6f8a2"}
{"level":"info","ts":"2026-09-18T10:01:16.737Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"30af3134-a980-4c6e-aa99-258736d6f8a2","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:16.737Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"30af3134-a980-4c6e-aa99-258736d6f8a2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:19.737Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f58b1a70-4cc1-42ed-94ed-3ab6ca9a1847","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:19.737Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f58b1a70-4cc1-42ed-94ed-3ab6ca9a1847","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:19.738Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f58b1a70-4cc1-42ed-94ed-3ab6ca9a1847","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:19.738Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f58b1a70-4cc1-42ed-94ed-3ab6ca9a1847"}
{"level":"info","ts":"2026-09-18T10:01:19.738Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f58b1a70-4cc1-42ed-94ed-3ab6ca9a1847","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:19.738Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f58b1a70-4cc1-42ed-94ed-3ab6ca9a1847","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:22.738Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"36cb4b3c-7d41-4fb0-89c3-1cd15e473887","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:22.738Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"36cb4b3c-7d41-4fb0-89c3-1cd15e473887","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:22.738Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"36cb4b3c-7d41-4fb0-89c3-1cd15e473887","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:22.738Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"36cb4b3c-7d41-4fb0-89c3-1cd15e473887"}
{"level":"info","ts":"2026-09-18T10:01:22.738Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"36cb4b3c-7d41-4fb0-89c3-1cd15e473887","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:22.738Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"36cb4b3c-7d41-4fb0-89c3-1cd15e473887","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:25.739Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"47092cf7-a5f3-45b4-bdb6-1584e15b6502","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:25.739Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"47092cf7-a5f3-45b4-bdb6-1584e15b6502","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:25.739Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"47092cf7-a5f3-45b4-bdb6-1584e15b6502","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:25.739Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"47092cf7-a5f3-45b4-bdb6-1584e15b6502"}
{"level":"info","ts":"2026-09-18T10:01:25.739Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"47092cf7-a5f3-45b4-bdb6-1584e15b6502","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:25.739Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"47092cf7-a5f3-45b4-bdb6-1584e15b6502","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:28.740Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"79f80bf1-8f34-4462-ada0-ca0ca6c47704","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:28.740Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"79f80bf1-8f34-4462-ada0-ca0ca6c47704","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:28.740Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"79f80bf1-8f34-4462-ada0-ca0ca6c47704","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:28.740Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"79f80bf1-8f34-4462-ada0-ca0ca6c47704"}
{"level":"info","ts":"2026-09-18T10:01:28.740Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"79f80bf1-8f34-4462-ada0-ca0ca6c47704","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:28.740Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"79f80bf1-8f34-4462-ada0-ca0ca6c47704","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:31.070Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"b70fa1d3-9dc0-44fe-8869-1a0c51c10739","namespace":"test-ns-ltx56","clusterName":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:01:31.070Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-q64ml","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"fail-k8s-job-q64ml","reconcileID":"b70fa1d3-9dc0-44fe-8869-1a0c51c10739","namespace":"test-ns-ltx56","clusterName":"fail-k8s-job-q64ml"}
{"level":"info","ts":"2026-09-18T10:01:31.741Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b5c4835a-8f6d-4ff6-9d32-feefb6fde611","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:31.741Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b5c4835a-8f6d-4ff6-9d32-feefb6fde611","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:31.741Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b5c4835a-8f6d-4ff6-9d32-feefb6fde611","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:31.741Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b5c4835a-8f6d-4ff6-9d32-feefb6fde611"}
{"level":"info","ts":"2026-09-18T10:01:31.741Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b5c4835a-8f6d-4ff6-9d32-feefb6fde611","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:31.741Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b5c4835a-8f6d-4ff6-9d32-feefb6fde611","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:34.742Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6bad945d-bafd-444e-aa04-da68cefc375c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:34.742Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6bad945d-bafd-444e-aa04-da68cefc375c","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:34.742Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6bad945d-bafd-444e-aa04-da68cefc375c","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:34.742Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6bad945d-bafd-444e-aa04-da68cefc375c"}
{"level":"info","ts":"2026-09-18T10:01:34.742Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6bad945d-bafd-444e-aa04-da68cefc375c","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:34.742Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6bad945d-bafd-444e-aa04-da68cefc375c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:37.743Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6688d31d-38ff-4fe0-81ca-4efe5029e53b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:37.743Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6688d31d-38ff-4fe0-81ca-4efe5029e53b","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:37.743Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6688d31d-38ff-4fe0-81ca-4efe5029e53b","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:37.743Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6688d31d-38ff-4fe0-81ca-4efe5029e53b"}
{"level":"info","ts":"2026-09-18T10:01:37.743Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6688d31d-38ff-4fe0-81ca-4efe5029e53b","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:37.743Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6688d31d-38ff-4fe0-81ca-4efe5029e53b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:40.744Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"65cb62df-cd77-4eb5-b63f-c4673c759caf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:40.744Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"65cb62df-cd77-4eb5-b63f-c4673c759caf","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:40.744Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"65cb62df-cd77-4eb5-b63f-c4673c759caf","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:40.744Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"65cb62df-cd77-4eb5-b63f-c4673c759caf"}
{"level":"info","ts":"2026-09-18T10:01:40.744Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"65cb62df-cd77-4eb5-b63f-c4673c759caf","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:40.744Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"65cb62df-cd77-4eb5-b63f-c4673c759caf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:43.745Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d866926a-6158-415f-8444-61eef2339aef","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:43.745Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d866926a-6158-415f-8444-61eef2339aef","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:43.745Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d866926a-6158-415f-8444-61eef2339aef","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:43.745Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d866926a-6158-415f-8444-61eef2339aef"}
{"level":"info","ts":"2026-09-18T10:01:43.745Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d866926a-6158-415f-8444-61eef2339aef","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:43.745Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d866926a-6158-415f-8444-61eef2339aef","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:46.747Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f5c8a98d-70c5-4065-9dc9-020baf7341f7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:46.747Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f5c8a98d-70c5-4065-9dc9-020baf7341f7","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:46.747Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f5c8a98d-70c5-4065-9dc9-020baf7341f7","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:46.747Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f5c8a98d-70c5-4065-9dc9-020baf7341f7"}
{"level":"info","ts":"2026-09-18T10:01:46.747Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f5c8a98d-70c5-4065-9dc9-020baf7341f7","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:46.747Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"f5c8a98d-70c5-4065-9dc9-020baf7341f7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:49.748Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6d1e4d2c-102a-47c8-8017-9521f9f69ae1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:49.748Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6d1e4d2c-102a-47c8-8017-9521f9f69ae1","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:49.748Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6d1e4d2c-102a-47c8-8017-9521f9f69ae1","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:49.748Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6d1e4d2c-102a-47c8-8017-9521f9f69ae1"}
{"level":"info","ts":"2026-09-18T10:01:49.748Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6d1e4d2c-102a-47c8-8017-9521f9f69ae1","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:49.748Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6d1e4d2c-102a-47c8-8017-9521f9f69ae1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:52.748Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"644f1f83-53ff-4a38-a20c-03fb0553bf7d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:52.748Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"644f1f83-53ff-4a38-a20c-03fb0553bf7d","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:52.748Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"644f1f83-53ff-4a38-a20c-03fb0553bf7d","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:52.748Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"644f1f83-53ff-4a38-a20c-03fb0553bf7d"}
{"level":"info","ts":"2026-09-18T10:01:52.748Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"644f1f83-53ff-4a38-a20c-03fb0553bf7d","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:52.748Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"644f1f83-53ff-4a38-a20c-03fb0553bf7d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:55.749Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"938abd85-cd60-4de9-b02b-597ff24a14f1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:55.749Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"938abd85-cd60-4de9-b02b-597ff24a14f1","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:55.749Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"938abd85-cd60-4de9-b02b-597ff24a14f1","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:55.749Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"938abd85-cd60-4de9-b02b-597ff24a14f1"}
{"level":"info","ts":"2026-09-18T10:01:55.749Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"938abd85-cd60-4de9-b02b-597ff24a14f1","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:55.749Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"938abd85-cd60-4de9-b02b-597ff24a14f1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:01:58.749Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"70bd7949-a57e-4121-8a63-1cd580d0dac1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:01:58.749Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"70bd7949-a57e-4121-8a63-1cd580d0dac1","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:01:58.749Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"70bd7949-a57e-4121-8a63-1cd580d0dac1","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:01:58.750Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"70bd7949-a57e-4121-8a63-1cd580d0dac1"}
{"level":"info","ts":"2026-09-18T10:01:58.750Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"70bd7949-a57e-4121-8a63-1cd580d0dac1","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:01:58.750Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"70bd7949-a57e-4121-8a63-1cd580d0dac1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:01.750Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"a0dcc683-471e-4241-a3c2-8107610a3aa3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:01.750Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"a0dcc683-471e-4241-a3c2-8107610a3aa3","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:01.750Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"a0dcc683-471e-4241-a3c2-8107610a3aa3","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:01.750Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"a0dcc683-471e-4241-a3c2-8107610a3aa3"}
{"level":"info","ts":"2026-09-18T10:02:01.750Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"a0dcc683-471e-4241-a3c2-8107610a3aa3","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:02:01.750Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"a0dcc683-471e-4241-a3c2-8107610a3aa3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:04.752Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"55625e56-093f-4cf6-bab6-1dd8e87de726","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:04.752Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"55625e56-093f-4cf6-bab6-1dd8e87de726","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:04.752Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"55625e56-093f-4cf6-bab6-1dd8e87de726","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:04.752Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"55625e56-093f-4cf6-bab6-1dd8e87de726"}
{"level":"info","ts":"2026-09-18T10:02:04.752Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"55625e56-093f-4cf6-bab6-1dd8e87de726","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:02:04.752Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"55625e56-093f-4cf6-bab6-1dd8e87de726","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:07.752Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"cba02080-f1b7-49ab-b7a2-b0c108ae47f2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:07.752Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"cba02080-f1b7-49ab-b7a2-b0c108ae47f2","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:07.752Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"cba02080-f1b7-49ab-b7a2-b0c108ae47f2","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:07.752Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"cba02080-f1b7-49ab-b7a2-b0c108ae47f2"}
{"level":"info","ts":"2026-09-18T10:02:07.752Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"cba02080-f1b7-49ab-b7a2-b0c108ae47f2","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:02:07.752Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"cba02080-f1b7-49ab-b7a2-b0c108ae47f2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:10.753Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"2432d405-7e78-4528-b27d-8f279361392f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:10.753Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"2432d405-7e78-4528-b27d-8f279361392f","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:10.753Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"2432d405-7e78-4528-b27d-8f279361392f","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:10.753Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"2432d405-7e78-4528-b27d-8f279361392f"}
{"level":"info","ts":"2026-09-18T10:02:10.753Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"2432d405-7e78-4528-b27d-8f279361392f","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:02:10.753Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"2432d405-7e78-4528-b27d-8f279361392f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:12.905Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"bf309ead-d16b-44f2-8d13-e442fb089d77"}
{"level":"info","ts":"2026-09-18T10:02:12.905Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"bf309ead-d16b-44f2-8d13-e442fb089d77"}
{"level":"info","ts":"2026-09-18T10:02:12.905Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"bf309ead-d16b-44f2-8d13-e442fb089d77"}
{"level":"info","ts":"2026-09-18T10:02:12.905Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"bf309ead-d16b-44f2-8d13-e442fb089d77","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:02:12.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"bf309ead-d16b-44f2-8d13-e442fb089d77","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:02:12.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"bf309ead-d16b-44f2-8d13-e442fb089d77","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:02:12.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"bf309ead-d16b-44f2-8d13-e442fb089d77","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:02:12.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"bf309ead-d16b-44f2-8d13-e442fb089d77","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:02:12.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"bf309ead-d16b-44f2-8d13-e442fb089d77","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:02:12.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"bf309ead-d16b-44f2-8d13-e442fb089d77","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:02:12.906Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"bf309ead-d16b-44f2-8d13-e442fb089d77","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:02:12.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"bf309ead-d16b-44f2-8d13-e442fb089d77","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:02:12.906Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"bf309ead-d16b-44f2-8d13-e442fb089d77","podName":"stop-9cztm-small-group-worker-w9ml2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:02:12.906Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"bf309ead-d16b-44f2-8d13-e442fb089d77","podName":"stop-9cztm-small-group-worker-w9ml2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:02:12.906Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"bf309ead-d16b-44f2-8d13-e442fb089d77","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:02:12.906Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"bf309ead-d16b-44f2-8d13-e442fb089d77","seconds":300}
{"level":"info","ts":"2026-09-18T10:02:13.753Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0b8699c4-1bbd-4736-a979-e24bc1b34333","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:13.753Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0b8699c4-1bbd-4736-a979-e24bc1b34333","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:13.753Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0b8699c4-1bbd-4736-a979-e24bc1b34333","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:13.754Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0b8699c4-1bbd-4736-a979-e24bc1b34333"}
{"level":"info","ts":"2026-09-18T10:02:13.754Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0b8699c4-1bbd-4736-a979-e24bc1b34333","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:02:13.754Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0b8699c4-1bbd-4736-a979-e24bc1b34333","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:13.916Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"b0906fcc-11cd-42d3-80cd-3a9d98d98cf5"}
{"level":"info","ts":"2026-09-18T10:02:13.916Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"b0906fcc-11cd-42d3-80cd-3a9d98d98cf5"}
{"level":"info","ts":"2026-09-18T10:02:13.916Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"b0906fcc-11cd-42d3-80cd-3a9d98d98cf5"}
{"level":"info","ts":"2026-09-18T10:02:13.916Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"b0906fcc-11cd-42d3-80cd-3a9d98d98cf5","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:02:13.916Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"b0906fcc-11cd-42d3-80cd-3a9d98d98cf5","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:02:13.916Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"b0906fcc-11cd-42d3-80cd-3a9d98d98cf5","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:02:13.916Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"b0906fcc-11cd-42d3-80cd-3a9d98d98cf5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:02:13.916Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"b0906fcc-11cd-42d3-80cd-3a9d98d98cf5","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:02:13.916Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"b0906fcc-11cd-42d3-80cd-3a9d98d98cf5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:02:13.916Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"b0906fcc-11cd-42d3-80cd-3a9d98d98cf5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:02:13.916Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"b0906fcc-11cd-42d3-80cd-3a9d98d98cf5","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:02:13.916Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"b0906fcc-11cd-42d3-80cd-3a9d98d98cf5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:02:13.916Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"b0906fcc-11cd-42d3-80cd-3a9d98d98cf5","podName":"stop-9cztm-small-group-worker-w9ml2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:02:13.916Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"b0906fcc-11cd-42d3-80cd-3a9d98d98cf5","podName":"stop-9cztm-small-group-worker-w9ml2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:02:13.916Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"b0906fcc-11cd-42d3-80cd-3a9d98d98cf5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:02:13.916Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"b0906fcc-11cd-42d3-80cd-3a9d98d98cf5","seconds":300}
{"level":"info","ts":"2026-09-18T10:02:14.109Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"0582ef5b-0b3e-4320-b3d3-983bbced4d5c","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:14.109Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"0582ef5b-0b3e-4320-b3d3-983bbced4d5c","pod":"stop-9cztm-head-jvvdj","ip":"10.133.0.41"}
{"level":"info","ts":"2026-09-18T10:02:14.109Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"0582ef5b-0b3e-4320-b3d3-983bbced4d5c","pod":"stop-9cztm-small-group-worker-w9ml2","ip":"10.132.0.45"}
{"level":"info","ts":"2026-09-18T10:02:14.109Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"0582ef5b-0b3e-4320-b3d3-983bbced4d5c","rayCluster":"stop-9cztm","podIPs":["10.133.0.41","10.132.0.45"]}
{"level":"info","ts":"2026-09-18T10:02:16.754Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8e6d0273-f761-41a7-87a5-65e79c92b247","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:16.754Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8e6d0273-f761-41a7-87a5-65e79c92b247","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:16.754Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8e6d0273-f761-41a7-87a5-65e79c92b247","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:16.754Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8e6d0273-f761-41a7-87a5-65e79c92b247"}
{"level":"info","ts":"2026-09-18T10:02:16.754Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8e6d0273-f761-41a7-87a5-65e79c92b247","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:02:16.754Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8e6d0273-f761-41a7-87a5-65e79c92b247","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:19.755Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b30ba8b4-a607-4189-b8b4-af5ae953fa80","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:19.755Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b30ba8b4-a607-4189-b8b4-af5ae953fa80","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:19.755Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b30ba8b4-a607-4189-b8b4-af5ae953fa80","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:19.755Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b30ba8b4-a607-4189-b8b4-af5ae953fa80"}
{"level":"info","ts":"2026-09-18T10:02:19.755Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b30ba8b4-a607-4189-b8b4-af5ae953fa80","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:02:19.755Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b30ba8b4-a607-4189-b8b4-af5ae953fa80","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:22.756Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"85f967c0-8b83-4604-8454-3790d89c7e72","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:22.756Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"85f967c0-8b83-4604-8454-3790d89c7e72","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:22.756Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"85f967c0-8b83-4604-8454-3790d89c7e72","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:22.757Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"85f967c0-8b83-4604-8454-3790d89c7e72"}
{"level":"info","ts":"2026-09-18T10:02:22.757Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"85f967c0-8b83-4604-8454-3790d89c7e72","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:02:22.757Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"85f967c0-8b83-4604-8454-3790d89c7e72","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:22.981Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d874e22f-d1d4-46c0-bc19-48159fceac7e"}
{"level":"info","ts":"2026-09-18T10:02:22.981Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d874e22f-d1d4-46c0-bc19-48159fceac7e"}
{"level":"info","ts":"2026-09-18T10:02:22.981Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d874e22f-d1d4-46c0-bc19-48159fceac7e"}
{"level":"info","ts":"2026-09-18T10:02:22.981Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d874e22f-d1d4-46c0-bc19-48159fceac7e","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:02:22.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d874e22f-d1d4-46c0-bc19-48159fceac7e","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:02:22.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d874e22f-d1d4-46c0-bc19-48159fceac7e","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:02:22.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d874e22f-d1d4-46c0-bc19-48159fceac7e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:02:22.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d874e22f-d1d4-46c0-bc19-48159fceac7e","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:02:22.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d874e22f-d1d4-46c0-bc19-48159fceac7e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:02:22.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d874e22f-d1d4-46c0-bc19-48159fceac7e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:02:22.981Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d874e22f-d1d4-46c0-bc19-48159fceac7e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:02:22.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d874e22f-d1d4-46c0-bc19-48159fceac7e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:02:22.981Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d874e22f-d1d4-46c0-bc19-48159fceac7e","podName":"stop-9cztm-small-group-worker-w9ml2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:02:22.981Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d874e22f-d1d4-46c0-bc19-48159fceac7e","podName":"stop-9cztm-small-group-worker-w9ml2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:02:22.982Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d874e22f-d1d4-46c0-bc19-48159fceac7e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:02:22.982Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"d874e22f-d1d4-46c0-bc19-48159fceac7e","seconds":300}
{"level":"info","ts":"2026-09-18T10:02:23.977Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1a080307-cfba-448c-96ea-dfc0e6590c49"}
{"level":"info","ts":"2026-09-18T10:02:23.977Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1a080307-cfba-448c-96ea-dfc0e6590c49"}
{"level":"info","ts":"2026-09-18T10:02:23.977Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1a080307-cfba-448c-96ea-dfc0e6590c49"}
{"level":"info","ts":"2026-09-18T10:02:23.977Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1a080307-cfba-448c-96ea-dfc0e6590c49","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:02:23.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1a080307-cfba-448c-96ea-dfc0e6590c49","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:02:23.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1a080307-cfba-448c-96ea-dfc0e6590c49","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:02:23.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1a080307-cfba-448c-96ea-dfc0e6590c49","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:02:23.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1a080307-cfba-448c-96ea-dfc0e6590c49","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:02:23.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1a080307-cfba-448c-96ea-dfc0e6590c49","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:02:23.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1a080307-cfba-448c-96ea-dfc0e6590c49","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:02:23.977Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1a080307-cfba-448c-96ea-dfc0e6590c49","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:02:23.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1a080307-cfba-448c-96ea-dfc0e6590c49","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:02:23.978Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1a080307-cfba-448c-96ea-dfc0e6590c49","podName":"stop-9cztm-small-group-worker-w9ml2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:00:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:02:23.978Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1a080307-cfba-448c-96ea-dfc0e6590c49","podName":"stop-9cztm-small-group-worker-w9ml2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:00:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:02:23.978Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"1a080307-cfba-448c-96ea-dfc0e6590c49","name":"stop-9cztm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:02:24.005Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"de6caf83-5a7b-41c8-b8dd-a893bb795fa5","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:24.005Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"67816221-7e21-4cb4-92ef-49e1aef7d2e4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:24.005Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"67816221-7e21-4cb4-92ef-49e1aef7d2e4","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:24.005Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"67816221-7e21-4cb4-92ef-49e1aef7d2e4","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:24.005Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"67816221-7e21-4cb4-92ef-49e1aef7d2e4"}
{"level":"info","ts":"2026-09-18T10:02:24.005Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"67816221-7e21-4cb4-92ef-49e1aef7d2e4","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:02:24.005Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"67816221-7e21-4cb4-92ef-49e1aef7d2e4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:01:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:24.005Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"67816221-7e21-4cb4-92ef-49e1aef7d2e4","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:02:24.005Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"fad85266-4c07-4d3c-a2e4-e94336051b51","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:24.005Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"fad85266-4c07-4d3c-a2e4-e94336051b51","pod":"stop-9cztm-head-jvvdj","ip":"10.133.0.41"}
{"level":"info","ts":"2026-09-18T10:02:24.005Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"fad85266-4c07-4d3c-a2e4-e94336051b51","pod":"stop-9cztm-small-group-worker-w9ml2","ip":"10.132.0.45"}
{"level":"info","ts":"2026-09-18T10:02:24.005Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"fad85266-4c07-4d3c-a2e4-e94336051b51","rayCluster":"stop-9cztm","podIPs":["10.133.0.41","10.132.0.45"]}
{"level":"info","ts":"2026-09-18T10:02:24.029Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bb16a483-11c3-4b51-8ba8-bf330c5d7a5c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:24.029Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bb16a483-11c3-4b51-8ba8-bf330c5d7a5c","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:24.029Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bb16a483-11c3-4b51-8ba8-bf330c5d7a5c","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:24.029Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bb16a483-11c3-4b51-8ba8-bf330c5d7a5c"}
{"level":"info","ts":"2026-09-18T10:02:24.029Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bb16a483-11c3-4b51-8ba8-bf330c5d7a5c","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:02:24.029Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bb16a483-11c3-4b51-8ba8-bf330c5d7a5c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:24.044Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"de6caf83-5a7b-41c8-b8dd-a893bb795fa5","name":"stop-9cztm-head"}
{"level":"info","ts":"2026-09-18T10:02:24.065Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"de6caf83-5a7b-41c8-b8dd-a893bb795fa5","name":"stop-9cztm-workers"}
{"level":"info","ts":"2026-09-18T10:02:24.065Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"de6caf83-5a7b-41c8-b8dd-a893bb795fa5","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:25.757Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d18fcdae-be67-4cab-a871-186f55eff8bb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:25.757Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d18fcdae-be67-4cab-a871-186f55eff8bb","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:25.757Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d18fcdae-be67-4cab-a871-186f55eff8bb","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:25.757Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d18fcdae-be67-4cab-a871-186f55eff8bb"}
{"level":"info","ts":"2026-09-18T10:02:25.757Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d18fcdae-be67-4cab-a871-186f55eff8bb","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:02:25.757Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"d18fcdae-be67-4cab-a871-186f55eff8bb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:26.004Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"97e20965-9ff8-4134-9cdb-ac46423efbd0"}
{"level":"info","ts":"2026-09-18T10:02:26.004Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"97e20965-9ff8-4134-9cdb-ac46423efbd0"}
{"level":"info","ts":"2026-09-18T10:02:26.004Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"97e20965-9ff8-4134-9cdb-ac46423efbd0"}
{"level":"info","ts":"2026-09-18T10:02:26.004Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"97e20965-9ff8-4134-9cdb-ac46423efbd0","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:02:26.004Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"97e20965-9ff8-4134-9cdb-ac46423efbd0","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:02:26.004Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"97e20965-9ff8-4134-9cdb-ac46423efbd0","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:02:26.004Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"97e20965-9ff8-4134-9cdb-ac46423efbd0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:02:26.004Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"97e20965-9ff8-4134-9cdb-ac46423efbd0","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:02:26.004Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"97e20965-9ff8-4134-9cdb-ac46423efbd0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:02:26.004Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"97e20965-9ff8-4134-9cdb-ac46423efbd0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:02:26.004Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"97e20965-9ff8-4134-9cdb-ac46423efbd0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:02:26.004Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"97e20965-9ff8-4134-9cdb-ac46423efbd0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:02:26.004Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"97e20965-9ff8-4134-9cdb-ac46423efbd0","podName":"stop-9cztm-small-group-worker-w9ml2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:00:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:02:26.004Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"97e20965-9ff8-4134-9cdb-ac46423efbd0","podName":"stop-9cztm-small-group-worker-w9ml2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:00:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:02:26.004Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"97e20965-9ff8-4134-9cdb-ac46423efbd0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:02:26.004Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"97e20965-9ff8-4134-9cdb-ac46423efbd0","seconds":300}
{"level":"info","ts":"2026-09-18T10:02:28.758Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e6eab348-bef9-4dd1-880e-1e4cb5e32458","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:28.758Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e6eab348-bef9-4dd1-880e-1e4cb5e32458","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:28.758Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e6eab348-bef9-4dd1-880e-1e4cb5e32458","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:28.759Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e6eab348-bef9-4dd1-880e-1e4cb5e32458"}
{"level":"info","ts":"2026-09-18T10:02:28.759Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e6eab348-bef9-4dd1-880e-1e4cb5e32458","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:02:28.759Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e6eab348-bef9-4dd1-880e-1e4cb5e32458","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:31.759Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8f319841-5571-4d68-8471-1adf00a405be","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:31.759Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8f319841-5571-4d68-8471-1adf00a405be","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:31.759Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8f319841-5571-4d68-8471-1adf00a405be","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:31.759Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8f319841-5571-4d68-8471-1adf00a405be"}
{"level":"info","ts":"2026-09-18T10:02:31.759Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8f319841-5571-4d68-8471-1adf00a405be","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:02:31.759Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8f319841-5571-4d68-8471-1adf00a405be","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:34.760Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e6ab7fe0-6005-4f72-84d3-f1580e1b6236","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:34.760Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e6ab7fe0-6005-4f72-84d3-f1580e1b6236","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:34.760Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e6ab7fe0-6005-4f72-84d3-f1580e1b6236","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:34.760Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e6ab7fe0-6005-4f72-84d3-f1580e1b6236"}
{"level":"info","ts":"2026-09-18T10:02:34.760Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e6ab7fe0-6005-4f72-84d3-f1580e1b6236","RayCluster":"stop-9cztm","State":""}
{"level":"info","ts":"2026-09-18T10:02:34.760Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e6ab7fe0-6005-4f72-84d3-f1580e1b6236","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:35.005Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"ea8beed5-09cb-4dc4-92a5-6d68021a2c6c"}
{"level":"info","ts":"2026-09-18T10:02:35.005Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"ea8beed5-09cb-4dc4-92a5-6d68021a2c6c"}
{"level":"info","ts":"2026-09-18T10:02:35.005Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"ea8beed5-09cb-4dc4-92a5-6d68021a2c6c"}
{"level":"info","ts":"2026-09-18T10:02:35.005Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"ea8beed5-09cb-4dc4-92a5-6d68021a2c6c","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:02:35.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"ea8beed5-09cb-4dc4-92a5-6d68021a2c6c","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:02:35.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"ea8beed5-09cb-4dc4-92a5-6d68021a2c6c","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:02:35.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"ea8beed5-09cb-4dc4-92a5-6d68021a2c6c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:02:35.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"ea8beed5-09cb-4dc4-92a5-6d68021a2c6c","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:02:35.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"ea8beed5-09cb-4dc4-92a5-6d68021a2c6c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:02:35.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"ea8beed5-09cb-4dc4-92a5-6d68021a2c6c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:02:35.005Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"ea8beed5-09cb-4dc4-92a5-6d68021a2c6c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:02:35.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"ea8beed5-09cb-4dc4-92a5-6d68021a2c6c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:02:35.005Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"ea8beed5-09cb-4dc4-92a5-6d68021a2c6c","name":"stop-9cztm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:02:35.029Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"12981ede-e5cb-46ba-b224-ed8134170430","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:35.029Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"9b3a46fe-3f02-4410-8382-d1f5550c607a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:35.029Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"9b3a46fe-3f02-4410-8382-d1f5550c607a","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:35.029Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"9b3a46fe-3f02-4410-8382-d1f5550c607a","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:35.029Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"9b3a46fe-3f02-4410-8382-d1f5550c607a"}
{"level":"info","ts":"2026-09-18T10:02:35.030Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"10481e6f-28ed-4a2c-a598-b05a173ff45d","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:35.030Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"10481e6f-28ed-4a2c-a598-b05a173ff45d","pod":"stop-9cztm-head-jvvdj","ip":"10.133.0.41"}
{"level":"info","ts":"2026-09-18T10:02:35.030Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"10481e6f-28ed-4a2c-a598-b05a173ff45d","pod":"stop-9cztm-small-group-worker-w9ml2","ip":"10.132.0.45"}
{"level":"info","ts":"2026-09-18T10:02:35.030Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"10481e6f-28ed-4a2c-a598-b05a173ff45d","rayCluster":"stop-9cztm","podIPs":["10.133.0.41","10.132.0.45"]}
{"level":"info","ts":"2026-09-18T10:02:35.054Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"9b3a46fe-3f02-4410-8382-d1f5550c607a","Kubernetes Job":"stop"}
{"level":"info","ts":"2026-09-18T10:02:35.054Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"9b3a46fe-3f02-4410-8382-d1f5550c607a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:35.054Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"9b3a46fe-3f02-4410-8382-d1f5550c607a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:02:35.067Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"12981ede-e5cb-46ba-b224-ed8134170430","name":"stop-9cztm-head"}
{"level":"info","ts":"2026-09-18T10:02:35.083Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5dd4211d-ec2e-44df-8251-ffe179af3a8f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:35.083Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5dd4211d-ec2e-44df-8251-ffe179af3a8f","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:35.084Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5dd4211d-ec2e-44df-8251-ffe179af3a8f","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:35.084Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5dd4211d-ec2e-44df-8251-ffe179af3a8f"}
{"level":"info","ts":"2026-09-18T10:02:35.084Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5dd4211d-ec2e-44df-8251-ffe179af3a8f","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:02:35.105Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5dd4211d-ec2e-44df-8251-ffe179af3a8f","JobId":"stop-qqkhp","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:35.105Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5dd4211d-ec2e-44df-8251-ffe179af3a8f"}
{"level":"error","ts":"2026-09-18T10:02:35.105Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"5dd4211d-ec2e-44df-8251-ffe179af3a8f","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:35.105Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b3314a29-a4bf-40c1-b9c4-68438c75d23b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:35.105Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b3314a29-a4bf-40c1-b9c4-68438c75d23b","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:35.105Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b3314a29-a4bf-40c1-b9c4-68438c75d23b","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:35.105Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b3314a29-a4bf-40c1-b9c4-68438c75d23b"}
{"level":"info","ts":"2026-09-18T10:02:35.105Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b3314a29-a4bf-40c1-b9c4-68438c75d23b","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:02:35.107Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"12981ede-e5cb-46ba-b224-ed8134170430","name":"stop-9cztm-workers"}
{"level":"info","ts":"2026-09-18T10:02:35.108Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"12981ede-e5cb-46ba-b224-ed8134170430","cluster":"stop-9cztm"}
{"level":"error","ts":"2026-09-18T10:02:35.109Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b3314a29-a4bf-40c1-b9c4-68438c75d23b","JobId":"stop-qqkhp","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:35.109Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b3314a29-a4bf-40c1-b9c4-68438c75d23b"}
{"level":"error","ts":"2026-09-18T10:02:35.109Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b3314a29-a4bf-40c1-b9c4-68438c75d23b","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:35.110Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"eff3cd4d-61ea-4f8b-abef-0c329a5ceaaa","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:35.110Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"eff3cd4d-61ea-4f8b-abef-0c329a5ceaaa","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:35.110Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"eff3cd4d-61ea-4f8b-abef-0c329a5ceaaa","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:35.111Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"eff3cd4d-61ea-4f8b-abef-0c329a5ceaaa"}
{"level":"info","ts":"2026-09-18T10:02:35.111Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"eff3cd4d-61ea-4f8b-abef-0c329a5ceaaa","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:02:35.114Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"eff3cd4d-61ea-4f8b-abef-0c329a5ceaaa","JobId":"stop-qqkhp","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:35.115Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"eff3cd4d-61ea-4f8b-abef-0c329a5ceaaa"}
{"level":"error","ts":"2026-09-18T10:02:35.115Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"eff3cd4d-61ea-4f8b-abef-0c329a5ceaaa","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:35.131Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8cf26095-6838-4e54-9555-fbcc7d33e993","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:35.131Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8cf26095-6838-4e54-9555-fbcc7d33e993","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:35.131Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8cf26095-6838-4e54-9555-fbcc7d33e993","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:35.131Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8cf26095-6838-4e54-9555-fbcc7d33e993"}
{"level":"info","ts":"2026-09-18T10:02:35.131Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8cf26095-6838-4e54-9555-fbcc7d33e993","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:02:35.135Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8cf26095-6838-4e54-9555-fbcc7d33e993","JobId":"stop-qqkhp","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:35.136Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8cf26095-6838-4e54-9555-fbcc7d33e993"}
{"level":"error","ts":"2026-09-18T10:02:35.136Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8cf26095-6838-4e54-9555-fbcc7d33e993","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:35.136Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"65beacb9-c27c-4f82-9670-6593be2228f4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:35.136Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"65beacb9-c27c-4f82-9670-6593be2228f4","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:35.136Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"65beacb9-c27c-4f82-9670-6593be2228f4","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:35.136Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"65beacb9-c27c-4f82-9670-6593be2228f4"}
{"level":"info","ts":"2026-09-18T10:02:35.136Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"65beacb9-c27c-4f82-9670-6593be2228f4","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:02:35.170Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"65beacb9-c27c-4f82-9670-6593be2228f4","JobId":"stop-qqkhp","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:35.170Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"65beacb9-c27c-4f82-9670-6593be2228f4"}
{"level":"error","ts":"2026-09-18T10:02:35.170Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"65beacb9-c27c-4f82-9670-6593be2228f4","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:35.251Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"48681f79-8a14-487d-9276-1ab4a10fbf50","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:35.251Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"48681f79-8a14-487d-9276-1ab4a10fbf50","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:35.251Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"48681f79-8a14-487d-9276-1ab4a10fbf50","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:35.251Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"48681f79-8a14-487d-9276-1ab4a10fbf50"}
{"level":"info","ts":"2026-09-18T10:02:35.251Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"48681f79-8a14-487d-9276-1ab4a10fbf50","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:02:35.256Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"48681f79-8a14-487d-9276-1ab4a10fbf50","JobId":"stop-qqkhp","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:35.256Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"48681f79-8a14-487d-9276-1ab4a10fbf50"}
{"level":"error","ts":"2026-09-18T10:02:35.256Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"48681f79-8a14-487d-9276-1ab4a10fbf50","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:35.416Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bdb00792-7e93-44a9-9948-dddc782f87f1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:35.416Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bdb00792-7e93-44a9-9948-dddc782f87f1","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:35.416Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bdb00792-7e93-44a9-9948-dddc782f87f1","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:35.416Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bdb00792-7e93-44a9-9948-dddc782f87f1"}
{"level":"info","ts":"2026-09-18T10:02:35.416Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bdb00792-7e93-44a9-9948-dddc782f87f1","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:02:35.421Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bdb00792-7e93-44a9-9948-dddc782f87f1","JobId":"stop-qqkhp","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:35.421Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bdb00792-7e93-44a9-9948-dddc782f87f1"}
{"level":"error","ts":"2026-09-18T10:02:35.421Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bdb00792-7e93-44a9-9948-dddc782f87f1","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:35.741Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"01edef6c-075a-4433-9d55-a28d051c48d7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:35.741Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"01edef6c-075a-4433-9d55-a28d051c48d7","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:35.741Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"01edef6c-075a-4433-9d55-a28d051c48d7","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:35.742Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"01edef6c-075a-4433-9d55-a28d051c48d7"}
{"level":"info","ts":"2026-09-18T10:02:35.742Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"01edef6c-075a-4433-9d55-a28d051c48d7","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:02:35.747Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"01edef6c-075a-4433-9d55-a28d051c48d7","JobId":"stop-qqkhp","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:35.747Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"01edef6c-075a-4433-9d55-a28d051c48d7"}
{"level":"error","ts":"2026-09-18T10:02:35.747Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"01edef6c-075a-4433-9d55-a28d051c48d7","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:36.132Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"1fe065b7-6628-47a8-9aa0-aacb2e6636fc","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:36.132Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"1fe065b7-6628-47a8-9aa0-aacb2e6636fc","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:36.132Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"1fe065b7-6628-47a8-9aa0-aacb2e6636fc","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:36.132Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"1fe065b7-6628-47a8-9aa0-aacb2e6636fc"}
{"level":"info","ts":"2026-09-18T10:02:36.132Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"1fe065b7-6628-47a8-9aa0-aacb2e6636fc","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:02:36.137Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"1fe065b7-6628-47a8-9aa0-aacb2e6636fc","JobId":"stop-qqkhp","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:36.137Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"1fe065b7-6628-47a8-9aa0-aacb2e6636fc"}
{"level":"error","ts":"2026-09-18T10:02:36.137Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"1fe065b7-6628-47a8-9aa0-aacb2e6636fc","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:36.387Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6e787ded-6de2-4176-a71d-9e89f93f79d3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:36.387Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6e787ded-6de2-4176-a71d-9e89f93f79d3","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:36.387Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6e787ded-6de2-4176-a71d-9e89f93f79d3","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:36.387Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6e787ded-6de2-4176-a71d-9e89f93f79d3"}
{"level":"info","ts":"2026-09-18T10:02:36.387Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6e787ded-6de2-4176-a71d-9e89f93f79d3","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:02:36.392Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6e787ded-6de2-4176-a71d-9e89f93f79d3","JobId":"stop-qqkhp","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:36.392Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6e787ded-6de2-4176-a71d-9e89f93f79d3"}
{"level":"error","ts":"2026-09-18T10:02:36.392Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"6e787ded-6de2-4176-a71d-9e89f93f79d3","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:37.030Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"538b7e76-0286-4349-9db3-6cdbaf5100a1"}
{"level":"info","ts":"2026-09-18T10:02:37.030Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"538b7e76-0286-4349-9db3-6cdbaf5100a1"}
{"level":"info","ts":"2026-09-18T10:02:37.031Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"538b7e76-0286-4349-9db3-6cdbaf5100a1"}
{"level":"info","ts":"2026-09-18T10:02:37.031Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"538b7e76-0286-4349-9db3-6cdbaf5100a1","1 head service found":"stop-9cztm-head-svc"}
{"level":"info","ts":"2026-09-18T10:02:37.031Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"538b7e76-0286-4349-9db3-6cdbaf5100a1","Found 1 head Pod":"stop-9cztm-head-jvvdj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:02:37.031Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"538b7e76-0286-4349-9db3-6cdbaf5100a1","head Pod":"stop-9cztm-head-jvvdj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-9cztm-head-jvvdj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:02:37.031Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"538b7e76-0286-4349-9db3-6cdbaf5100a1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:02:37.031Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"538b7e76-0286-4349-9db3-6cdbaf5100a1","worker Pod":"stop-9cztm-small-group-worker-w9ml2","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-9cztm-small-group-worker-w9ml2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:02:37.031Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"538b7e76-0286-4349-9db3-6cdbaf5100a1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:02:37.031Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"538b7e76-0286-4349-9db3-6cdbaf5100a1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:02:37.031Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"538b7e76-0286-4349-9db3-6cdbaf5100a1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:02:37.031Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"538b7e76-0286-4349-9db3-6cdbaf5100a1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:02:37.031Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"538b7e76-0286-4349-9db3-6cdbaf5100a1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:02:37.031Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"538b7e76-0286-4349-9db3-6cdbaf5100a1","seconds":300}
{"level":"info","ts":"2026-09-18T10:02:38.953Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0d2d064f-f4b3-4663-b184-1a2852d19e5e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:38.953Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0d2d064f-f4b3-4663-b184-1a2852d19e5e","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:38.953Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0d2d064f-f4b3-4663-b184-1a2852d19e5e","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:38.953Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0d2d064f-f4b3-4663-b184-1a2852d19e5e"}
{"level":"info","ts":"2026-09-18T10:02:38.953Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0d2d064f-f4b3-4663-b184-1a2852d19e5e","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:02:38.958Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0d2d064f-f4b3-4663-b184-1a2852d19e5e","JobId":"stop-qqkhp","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:38.958Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0d2d064f-f4b3-4663-b184-1a2852d19e5e"}
{"level":"error","ts":"2026-09-18T10:02:38.958Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0d2d064f-f4b3-4663-b184-1a2852d19e5e","error":"Job stop-qqkhp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:02:44.078Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b985efe4-af5f-4594-acde-839844007978","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:44.078Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b985efe4-af5f-4594-acde-839844007978","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:44.078Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b985efe4-af5f-4594-acde-839844007978","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:44.079Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b985efe4-af5f-4594-acde-839844007978"}
{"level":"info","ts":"2026-09-18T10:02:44.079Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b985efe4-af5f-4594-acde-839844007978","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:02:44.084Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b985efe4-af5f-4594-acde-839844007978","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:44.084Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b985efe4-af5f-4594-acde-839844007978","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:02:44.102Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"673563cf-74fd-478a-8bf2-48d683078139","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:44.102Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"673563cf-74fd-478a-8bf2-48d683078139","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:44.102Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"673563cf-74fd-478a-8bf2-48d683078139","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:44.102Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"673563cf-74fd-478a-8bf2-48d683078139"}
{"level":"info","ts":"2026-09-18T10:02:44.103Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"673563cf-74fd-478a-8bf2-48d683078139","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:02:44.166Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"673563cf-74fd-478a-8bf2-48d683078139","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:47.103Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"4b60f742-920a-47b6-bec0-ba837c6abe8f","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:47.103Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"4b60f742-920a-47b6-bec0-ba837c6abe8f","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:47.103Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"4b60f742-920a-47b6-bec0-ba837c6abe8f","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:47.103Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"4b60f742-920a-47b6-bec0-ba837c6abe8f"}
{"level":"info","ts":"2026-09-18T10:02:47.103Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"4b60f742-920a-47b6-bec0-ba837c6abe8f","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:02:47.105Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"4b60f742-920a-47b6-bec0-ba837c6abe8f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:47.105Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"4b60f742-920a-47b6-bec0-ba837c6abe8f","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:02:47.156Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"833c3d00-db6b-45ff-aa56-76c3fab1704f","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:47.156Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"833c3d00-db6b-45ff-aa56-76c3fab1704f","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:47.156Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"833c3d00-db6b-45ff-aa56-76c3fab1704f","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:47.156Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"833c3d00-db6b-45ff-aa56-76c3fab1704f"}
{"level":"info","ts":"2026-09-18T10:02:47.156Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"833c3d00-db6b-45ff-aa56-76c3fab1704f","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:02:47.159Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"833c3d00-db6b-45ff-aa56-76c3fab1704f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:50.157Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bf4d1545-7b08-4311-86f9-9715ae8fd342","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:50.157Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bf4d1545-7b08-4311-86f9-9715ae8fd342","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:50.157Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bf4d1545-7b08-4311-86f9-9715ae8fd342","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:50.157Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bf4d1545-7b08-4311-86f9-9715ae8fd342"}
{"level":"info","ts":"2026-09-18T10:02:50.157Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bf4d1545-7b08-4311-86f9-9715ae8fd342","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:02:50.160Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bf4d1545-7b08-4311-86f9-9715ae8fd342","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:53.161Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0f8f438e-9e15-4003-b845-4e8a805fdbaa","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:53.161Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0f8f438e-9e15-4003-b845-4e8a805fdbaa","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:53.161Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0f8f438e-9e15-4003-b845-4e8a805fdbaa","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:53.161Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0f8f438e-9e15-4003-b845-4e8a805fdbaa"}
{"level":"info","ts":"2026-09-18T10:02:53.161Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0f8f438e-9e15-4003-b845-4e8a805fdbaa","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:02:53.164Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"0f8f438e-9e15-4003-b845-4e8a805fdbaa","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:56.165Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"2dcad918-82b5-4c60-a2af-ecce1b42017a","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:56.165Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"2dcad918-82b5-4c60-a2af-ecce1b42017a","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:56.165Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"2dcad918-82b5-4c60-a2af-ecce1b42017a","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:56.165Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"2dcad918-82b5-4c60-a2af-ecce1b42017a"}
{"level":"info","ts":"2026-09-18T10:02:56.165Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"2dcad918-82b5-4c60-a2af-ecce1b42017a","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:02:56.168Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"2dcad918-82b5-4c60-a2af-ecce1b42017a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:02:59.169Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"7a753cf9-31c9-4eb1-87c7-e568e3f85d3c","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:02:59.169Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"7a753cf9-31c9-4eb1-87c7-e568e3f85d3c","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:02:59.169Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"7a753cf9-31c9-4eb1-87c7-e568e3f85d3c","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:02:59.169Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"7a753cf9-31c9-4eb1-87c7-e568e3f85d3c"}
{"level":"info","ts":"2026-09-18T10:02:59.169Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"7a753cf9-31c9-4eb1-87c7-e568e3f85d3c","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:02:59.172Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"7a753cf9-31c9-4eb1-87c7-e568e3f85d3c","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:02.172Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bb485aa2-8eaa-4f0e-97ea-deea088dc63b","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:02.172Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bb485aa2-8eaa-4f0e-97ea-deea088dc63b","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:03:02.172Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bb485aa2-8eaa-4f0e-97ea-deea088dc63b","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:02.172Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bb485aa2-8eaa-4f0e-97ea-deea088dc63b"}
{"level":"info","ts":"2026-09-18T10:03:02.172Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bb485aa2-8eaa-4f0e-97ea-deea088dc63b","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:03:02.175Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"bb485aa2-8eaa-4f0e-97ea-deea088dc63b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:05.176Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"a07617d2-3fb0-478d-ae9d-5a701b19abc1","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:05.176Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"a07617d2-3fb0-478d-ae9d-5a701b19abc1","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:03:05.176Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"a07617d2-3fb0-478d-ae9d-5a701b19abc1","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:05.176Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"a07617d2-3fb0-478d-ae9d-5a701b19abc1"}
{"level":"info","ts":"2026-09-18T10:03:05.176Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"a07617d2-3fb0-478d-ae9d-5a701b19abc1","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:03:05.179Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"a07617d2-3fb0-478d-ae9d-5a701b19abc1","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:08.179Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8094857c-c8ac-4b74-8baa-3f8a08392d18","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:08.179Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8094857c-c8ac-4b74-8baa-3f8a08392d18","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:03:08.179Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8094857c-c8ac-4b74-8baa-3f8a08392d18","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:08.180Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8094857c-c8ac-4b74-8baa-3f8a08392d18"}
{"level":"info","ts":"2026-09-18T10:03:08.180Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8094857c-c8ac-4b74-8baa-3f8a08392d18","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:03:08.182Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8094857c-c8ac-4b74-8baa-3f8a08392d18","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z","endTime":"2026-09-18T10:03:05Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:08.182Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"8094857c-c8ac-4b74-8baa-3f8a08392d18","old JobStatus":"RUNNING","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:03:08.210Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b5c69958-3414-4b0c-86ce-12f30de88daf","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:08.210Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b5c69958-3414-4b0c-86ce-12f30de88daf","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:03:08.210Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b5c69958-3414-4b0c-86ce-12f30de88daf","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:08.210Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b5c69958-3414-4b0c-86ce-12f30de88daf"}
{"level":"info","ts":"2026-09-18T10:03:08.210Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b5c69958-3414-4b0c-86ce-12f30de88daf","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:03:08.212Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b5c69958-3414-4b0c-86ce-12f30de88daf","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z","endTime":"2026-09-18T10:03:05Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z","endTime":"2026-09-18T10:03:05Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:11.202Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"de6cc164-7eed-4a28-b718-7ebdb8cc30a8","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:11.202Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"de6cc164-7eed-4a28-b718-7ebdb8cc30a8","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:03:11.202Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"de6cc164-7eed-4a28-b718-7ebdb8cc30a8","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:11.202Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"de6cc164-7eed-4a28-b718-7ebdb8cc30a8"}
{"level":"info","ts":"2026-09-18T10:03:11.202Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"de6cc164-7eed-4a28-b718-7ebdb8cc30a8","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:03:11.205Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"de6cc164-7eed-4a28-b718-7ebdb8cc30a8","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z","endTime":"2026-09-18T10:03:05Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z","endTime":"2026-09-18T10:03:05Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:11.211Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"c9e10973-e8cb-4284-be88-9238859e67bc","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:11.211Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"c9e10973-e8cb-4284-be88-9238859e67bc","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:03:11.211Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"c9e10973-e8cb-4284-be88-9238859e67bc","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:11.211Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"c9e10973-e8cb-4284-be88-9238859e67bc"}
{"level":"info","ts":"2026-09-18T10:03:11.211Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"c9e10973-e8cb-4284-be88-9238859e67bc","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:03:11.213Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"c9e10973-e8cb-4284-be88-9238859e67bc","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z","endTime":"2026-09-18T10:03:05Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z","endTime":"2026-09-18T10:03:05Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:12.222Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fc039766-2927-491a-bc68-2c7058987d1d","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:12.222Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fc039766-2927-491a-bc68-2c7058987d1d","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:03:12.222Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fc039766-2927-491a-bc68-2c7058987d1d","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:12.222Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fc039766-2927-491a-bc68-2c7058987d1d"}
{"level":"info","ts":"2026-09-18T10:03:12.222Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fc039766-2927-491a-bc68-2c7058987d1d","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:03:12.225Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"fc039766-2927-491a-bc68-2c7058987d1d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z","endTime":"2026-09-18T10:03:05Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z","endTime":"2026-09-18T10:03:05Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:12.268Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e8c9c62c-3667-463f-bbba-ee0378d0be8a","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:12.268Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e8c9c62c-3667-463f-bbba-ee0378d0be8a","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:03:12.268Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e8c9c62c-3667-463f-bbba-ee0378d0be8a","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:12.268Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e8c9c62c-3667-463f-bbba-ee0378d0be8a"}
{"level":"info","ts":"2026-09-18T10:03:12.268Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e8c9c62c-3667-463f-bbba-ee0378d0be8a","rayCluster":"stop-9cztm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:03:12.270Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e8c9c62c-3667-463f-bbba-ee0378d0be8a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z","endTime":"2026-09-18T10:03:05Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:02:43Z","endTime":"2026-09-18T10:03:05Z"},"jobId":"stop-qqkhp","rayClusterName":"stop-9cztm","dashboardURL":"stop-9cztm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Complete","message":"Job was intentionally stopped.","startTime":"2026-09-18T10:00:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:02:35Z","stateTransitionTimes":{"ready":"2026-09-18T10:02:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.41","serviceIP":"10.133.0.41","podName":"stop-9cztm-head-jvvdj","serviceName":"stop-9cztm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:01:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:02:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:00:43Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:00:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:12.270Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e8c9c62c-3667-463f-bbba-ee0378d0be8a","old JobStatus":"STOPPED","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-09-18T10:03:12.297Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"e65db8ad-8468-49cb-9ec9-759011db126b","JobStatus":"STOPPED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:13.084Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"271cc88b-ba80-4dd5-8fc3-4759506ee949","DeletionTimestamp":"2026-09-18 10:03:13 +0000 UTC"}
{"level":"info","ts":"2026-09-18T10:03:13.084Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"271cc88b-ba80-4dd5-8fc3-4759506ee949","finalizer":"ray.io/rayjob-finalizer"}
{"level":"error","ts":"2026-09-18T10:03:13.106Z","logger":"controllers.RayJob","msg":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type types.RuntimeEnvType","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-ltx56"},"reconcileID":"38982785-6a66-4534-87cd-90cc4a2328f6","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type types.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.validateRayJob\n\t/workspace/controllers/ray/rayjob_controller.go:520\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:153\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:13.106Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-ltx56"},"reconcileID":"38982785-6a66-4534-87cd-90cc4a2328f6","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"ValidationFailed","reason":"ValidationFailed","message":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type types.RuntimeEnvType","rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:03:13.106Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-ltx56"},"reconcileID":"38982785-6a66-4534-87cd-90cc4a2328f6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"ValidationFailed"}
{"level":"info","ts":"2026-09-18T10:03:13.127Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"56755900-0413-4210-968e-21e6ba752d2e"}
{"level":"error","ts":"2026-09-18T10:03:13.127Z","logger":"controllers.RayJob","msg":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type types.RuntimeEnvType","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-ltx56"},"reconcileID":"53fc1ea0-ded5-401f-b5b7-36e05a45f129","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type types.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.validateRayJob\n\t/workspace/controllers/ray/rayjob_controller.go:520\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:153\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:13.127Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-ltx56"},"reconcileID":"53fc1ea0-ded5-401f-b5b7-36e05a45f129","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"ValidationFailed","reason":"ValidationFailed","message":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type types.RuntimeEnvType","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"ValidationFailed","reason":"ValidationFailed","message":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type types.RuntimeEnvType","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:03:13.142Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"104148c3-8fc6-4ea5-8bae-402de296152e"}
{"level":"info","ts":"2026-09-18T10:03:13.155Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"c0ffa0be-06da-40c5-9f24-c1306e244e01"}
{"level":"info","ts":"2026-09-18T10:03:13.158Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"032aab21-bb90-49e6-8bc4-a3d8082c644c","namespacedName":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:13.158Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"032aab21-bb90-49e6-8bc4-a3d8082c644c","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:03:13.158Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"032aab21-bb90-49e6-8bc4-a3d8082c644c","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:03:13.158Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"032aab21-bb90-49e6-8bc4-a3d8082c644c","namespace":"test-ns-ltx56","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T10:03:13.158Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"b59f3fa6-2430-404a-8f97-62fbe95f6490"}
{"level":"info","ts":"2026-09-18T10:03:13.158Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"reconcileID":"51bb0a62-61e5-4a9b-ba79-cb515b7357be"}
{"level":"info","ts":"2026-09-18T10:03:13.158Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"b82abe0d-3ced-4eb8-85b2-9880abfee22f","rayCluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:03:13.159Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"a7f7e9d4-9644-4a21-989c-791e419ce087"}
{"level":"info","ts":"2026-09-18T10:03:13.159Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"b82abe0d-3ced-4eb8-85b2-9880abfee22f","pod":"stop-9cztm-head-jvvdj","ip":"10.133.0.41"}
{"level":"info","ts":"2026-09-18T10:03:13.159Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"b82abe0d-3ced-4eb8-85b2-9880abfee22f","pod":"stop-9cztm-small-group-worker-w9ml2","ip":"10.132.0.45"}
{"level":"info","ts":"2026-09-18T10:03:13.159Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"b82abe0d-3ced-4eb8-85b2-9880abfee22f","rayCluster":"stop-9cztm","podIPs":["10.133.0.41","10.132.0.45"]}
{"level":"info","ts":"2026-09-18T10:03:13.175Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"032aab21-bb90-49e6-8bc4-a3d8082c644c","name":"kuberay-gateway-access","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:03:13.191Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"032aab21-bb90-49e6-8bc4-a3d8082c644c","name":"test-ns-ltx56-stop-9cztm","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T10:03:13.203Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"032aab21-bb90-49e6-8bc4-a3d8082c644c","configMap":"kube-rbac-proxy-config-stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:03:13.214Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"032aab21-bb90-49e6-8bc4-a3d8082c644c","serviceAccount":"stop-9cztm-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:03:13.288Z","logger":"raycluster-resource","msg":"default","name":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:03:13.288Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-9cztm","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:03:13.320Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"1d65c0cf-6272-44e1-a095-6937826db221","namespace":"test-ns-ltx56","clusterName":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:03:13.321Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"c3e20371-ccd9-419b-b2dd-ccf0b0a465fa"}
{"level":"info","ts":"2026-09-18T10:03:13.320Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"ba772efe-6398-4573-8dc0-13a4802c49d4"}
{"level":"info","ts":"2026-09-18T10:03:13.323Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"032aab21-bb90-49e6-8bc4-a3d8082c644c","cluster":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:03:13.323Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"441df93d-6fa3-47f1-8639-93ee3c5b6aab","namespacedName":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:13.323Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"441df93d-6fa3-47f1-8639-93ee3c5b6aab"}
{"level":"info","ts":"2026-09-18T10:03:13.323Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"441df93d-6fa3-47f1-8639-93ee3c5b6aab","namespace":"test-ns-ltx56","authClusters":0}
{"level":"info","ts":"2026-09-18T10:03:13.323Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"441df93d-6fa3-47f1-8639-93ee3c5b6aab","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:03:13.356Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"97f34251-9960-47c8-995d-4e2c5963e983"}
{"level":"info","ts":"2026-09-18T10:03:13.356Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"23446123-80b0-44f7-a4a8-3e1685023641","namespacedName":{"name":"stop-9cztm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:13.356Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"23446123-80b0-44f7-a4a8-3e1685023641"}
{"level":"info","ts":"2026-09-18T10:03:13.356Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"23446123-80b0-44f7-a4a8-3e1685023641","namespace":"test-ns-ltx56","authClusters":0}
{"level":"info","ts":"2026-09-18T10:03:13.356Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"23446123-80b0-44f7-a4a8-3e1685023641","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:03:13.360Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"a3373fe9-37a6-4386-a50d-45ec202d8ca0"}
{"level":"error","ts":"2026-09-18T10:03:13.383Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"1d65c0cf-6272-44e1-a095-6937826db221","certificate":"ray-head-cert-stop-9cztm","error":"certificates.cert-manager.io \"ray-head-cert-stop-9cztm\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:03:13.443Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"1d65c0cf-6272-44e1-a095-6937826db221","certificate":"ray-worker-cert-stop-9cztm","error":"certificates.cert-manager.io \"ray-worker-cert-stop-9cztm\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:13.481Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"1d65c0cf-6272-44e1-a095-6937826db221","errorCount":2}
{"level":"info","ts":"2026-09-18T10:03:13.481Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"1d65c0cf-6272-44e1-a095-6937826db221"}
{"level":"error","ts":"2026-09-18T10:03:13.481Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"1d65c0cf-6272-44e1-a095-6937826db221","error":"certificates.cert-manager.io \"ray-head-cert-stop-9cztm\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:13.486Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"4654fb61-795c-41de-a3a9-dfd513de4f14","namespace":"test-ns-ltx56","clusterName":"stop-9cztm"}
{"level":"info","ts":"2026-09-18T10:03:13.486Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-9cztm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"stop-9cztm","reconcileID":"4654fb61-795c-41de-a3a9-dfd513de4f14","namespace":"test-ns-ltx56","clusterName":"stop-9cztm"}
{"level":"error","ts":"2026-09-18T10:03:14.138Z","logger":"controllers.RayJob","msg":"The RayJob metadata is invalid: RayJob name should be no more than 47 characters","RayJob":{"name":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","namespace":"test-ns-ltx56"},"reconcileID":"84f29bcd-1424-4b46-9c96-922551634fff","error":"The RayJob metadata is invalid: RayJob name should be no more than 47 characters","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.validateRayJob\n\t/workspace/controllers/ray/rayjob_controller.go:520\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:153\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:14.138Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","namespace":"test-ns-ltx56"},"reconcileID":"84f29bcd-1424-4b46-9c96-922551634fff","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"ValidationFailed","reason":"ValidationFailed","message":"The RayJob metadata is invalid: RayJob name should be no more than 47 characters","rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:03:14.138Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","namespace":"test-ns-ltx56"},"reconcileID":"84f29bcd-1424-4b46-9c96-922551634fff","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"ValidationFailed"}
{"level":"error","ts":"2026-09-18T10:03:14.160Z","logger":"controllers.RayJob","msg":"The RayJob metadata is invalid: RayJob name should be no more than 47 characters","RayJob":{"name":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","namespace":"test-ns-ltx56"},"reconcileID":"8c8c4164-06f3-43dd-9b9e-f71ca08a90d1","error":"The RayJob metadata is invalid: RayJob name should be no more than 47 characters","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.validateRayJob\n\t/workspace/controllers/ray/rayjob_controller.go:520\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:153\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:14.160Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","namespace":"test-ns-ltx56"},"reconcileID":"8c8c4164-06f3-43dd-9b9e-f71ca08a90d1","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"ValidationFailed","reason":"ValidationFailed","message":"The RayJob metadata is invalid: RayJob name should be no more than 47 characters","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"ValidationFailed","reason":"ValidationFailed","message":"The RayJob metadata is invalid: RayJob name should be no more than 47 characters","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:03:14.213Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"stop","namespace":"test-ns-ltx56"},"reconcileID":"becd8e11-9d77-467c-90d9-524912654b91"}
{"level":"info","ts":"2026-09-18T10:03:15.172Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"3ebbc886-7543-4b03-b8cb-7870b10ebd75","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:15.172Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"3ebbc886-7543-4b03-b8cb-7870b10ebd75","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T10:03:15.186Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"3ebbc886-7543-4b03-b8cb-7870b10ebd75"}
{"level":"info","ts":"2026-09-18T10:03:15.186Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"3ebbc886-7543-4b03-b8cb-7870b10ebd75","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T10:03:15.186Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"3ebbc886-7543-4b03-b8cb-7870b10ebd75","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:03:15.186Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"3ebbc886-7543-4b03-b8cb-7870b10ebd75","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:03:15.200Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"9d108037-c5a0-4f2d-a9cf-2cdc6f52f433","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:15.200Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"9d108037-c5a0-4f2d-a9cf-2cdc6f52f433"}
{"level":"info","ts":"2026-09-18T10:03:15.200Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"9d108037-c5a0-4f2d-a9cf-2cdc6f52f433","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T10:03:15.200Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"9d108037-c5a0-4f2d-a9cf-2cdc6f52f433","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-jgtjg","rayClusterName":"long-running-z9zm4","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:03:15.200Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"9d108037-c5a0-4f2d-a9cf-2cdc6f52f433","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:03:15.209Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"9d108037-c5a0-4f2d-a9cf-2cdc6f52f433","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T10:03:15.209Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"9d108037-c5a0-4f2d-a9cf-2cdc6f52f433"}
{"level":"error","ts":"2026-09-18T10:03:15.209Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"9d108037-c5a0-4f2d-a9cf-2cdc6f52f433","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.209Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"c0dd70bb-99fc-4b82-a29e-cb8b605b924c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:15.209Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"c0dd70bb-99fc-4b82-a29e-cb8b605b924c","name":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.209Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"c0dd70bb-99fc-4b82-a29e-cb8b605b924c","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:15.209Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"c0dd70bb-99fc-4b82-a29e-cb8b605b924c","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:15.215Z","logger":"raycluster-resource","msg":"default","name":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.215Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-5qfl8","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:03:15.237Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"c0dd70bb-99fc-4b82-a29e-cb8b605b924c","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:15.237Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"7e5e46eb-3b63-40c8-a5b4-a45b60b5d4eb","namespacedName":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:15.237Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"7e5e46eb-3b63-40c8-a5b4-a45b60b5d4eb","mode":"IntegratedOAuth","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.237Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"7e5e46eb-3b63-40c8-a5b4-a45b60b5d4eb","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.237Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"7e5e46eb-3b63-40c8-a5b4-a45b60b5d4eb","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.237Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"c0dd70bb-99fc-4b82-a29e-cb8b605b924c"}
{"level":"info","ts":"2026-09-18T10:03:15.237Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"c0dd70bb-99fc-4b82-a29e-cb8b605b924c","RayCluster":"long-running-5qfl8","State":""}
{"level":"info","ts":"2026-09-18T10:03:15.237Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"c0dd70bb-99fc-4b82-a29e-cb8b605b924c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"ea41736f-619a-497c-a373-6f8df7d30b04","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"ea41736f-619a-497c-a373-6f8df7d30b04","name":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"ea41736f-619a-497c-a373-6f8df7d30b04","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"ea41736f-619a-497c-a373-6f8df7d30b04"}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"ea41736f-619a-497c-a373-6f8df7d30b04","RayCluster":"long-running-5qfl8","State":""}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f2ed0208-7a32-48e7-b5f8-8a8254801766"}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f2ed0208-7a32-48e7-b5f8-8a8254801766"}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"ea41736f-619a-497c-a373-6f8df7d30b04","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f2ed0208-7a32-48e7-b5f8-8a8254801766"}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"4a855e91-3f70-4a74-8202-e7cf13a822ab","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"4a855e91-3f70-4a74-8202-e7cf13a822ab","name":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"4a855e91-3f70-4a74-8202-e7cf13a822ab","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"40cf0ac4-5e51-4b61-a262-64a4fb67e979","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"40cf0ac4-5e51-4b61-a262-64a4fb67e979","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"4a855e91-3f70-4a74-8202-e7cf13a822ab"}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"4a855e91-3f70-4a74-8202-e7cf13a822ab","RayCluster":"long-running-5qfl8","State":""}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"4a855e91-3f70-4a74-8202-e7cf13a822ab","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:03:15.238Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"1820931f-03be-4d71-a909-1a1a8ed718a8","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.266Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f2ed0208-7a32-48e7-b5f8-8a8254801766","name":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:15.266Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f2ed0208-7a32-48e7-b5f8-8a8254801766","error":"head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"error","ts":"2026-09-18T10:03:15.266Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f2ed0208-7a32-48e7-b5f8-8a8254801766","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.266Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f2ed0208-7a32-48e7-b5f8-8a8254801766","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"info","ts":"2026-09-18T10:03:15.266Z","logger":"raycluster-resource","msg":"default","name":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f2ed0208-7a32-48e7-b5f8-8a8254801766","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.266Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-5qfl8","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:03:15.266Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f2ed0208-7a32-48e7-b5f8-8a8254801766","serviceSelectors":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/identifier":"long-running-5qfl8-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f2ed0208-7a32-48e7-b5f8-8a8254801766","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.266Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f2ed0208-7a32-48e7-b5f8-8a8254801766","error":"unable to find head service. cluster name long-running-5qfl8, filter labels [test-ns-ltx56 map[ray.io/cluster:long-running-5qfl8 ray.io/identifier:long-running-5qfl8-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-09-18T10:03:15.266Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f2ed0208-7a32-48e7-b5f8-8a8254801766"}
{"level":"error","ts":"2026-09-18T10:03:15.266Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f2ed0208-7a32-48e7-b5f8-8a8254801766","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.266Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"399bde19-c0fe-4f95-96a4-277bd289bf78"}
{"level":"info","ts":"2026-09-18T10:03:15.266Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"399bde19-c0fe-4f95-96a4-277bd289bf78"}
{"level":"info","ts":"2026-09-18T10:03:15.267Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"399bde19-c0fe-4f95-96a4-277bd289bf78"}
{"level":"info","ts":"2026-09-18T10:03:15.267Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"399bde19-c0fe-4f95-96a4-277bd289bf78","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:15.267Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"399bde19-c0fe-4f95-96a4-277bd289bf78","error":"head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"error","ts":"2026-09-18T10:03:15.267Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"399bde19-c0fe-4f95-96a4-277bd289bf78","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.267Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"399bde19-c0fe-4f95-96a4-277bd289bf78","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"info","ts":"2026-09-18T10:03:15.267Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"399bde19-c0fe-4f95-96a4-277bd289bf78","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.267Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"399bde19-c0fe-4f95-96a4-277bd289bf78","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.267Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"399bde19-c0fe-4f95-96a4-277bd289bf78","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.267Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"399bde19-c0fe-4f95-96a4-277bd289bf78","name":"long-running-5qfl8","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:03:15.291Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"1820931f-03be-4d71-a909-1a1a8ed718a8","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.291Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"359f5833-8492-4e6e-9167-81ea5be4c9d4","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"399bde19-c0fe-4f95-96a4-277bd289bf78"}
{"level":"error","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"399bde19-c0fe-4f95-96a4-277bd289bf78","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f28e207f-7373-4e7c-be7d-4a9a962778d4"}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f28e207f-7373-4e7c-be7d-4a9a962778d4"}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f28e207f-7373-4e7c-be7d-4a9a962778d4"}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"c2a88abd-1bec-4151-957e-d07dd2a80242","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"c2a88abd-1bec-4151-957e-d07dd2a80242","name":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"c2a88abd-1bec-4151-957e-d07dd2a80242","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f28e207f-7373-4e7c-be7d-4a9a962778d4","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f28e207f-7373-4e7c-be7d-4a9a962778d4","error":"head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"c2a88abd-1bec-4151-957e-d07dd2a80242"}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"c2a88abd-1bec-4151-957e-d07dd2a80242","RayCluster":"long-running-5qfl8","State":""}
{"level":"error","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f28e207f-7373-4e7c-be7d-4a9a962778d4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f28e207f-7373-4e7c-be7d-4a9a962778d4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"c2a88abd-1bec-4151-957e-d07dd2a80242","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"c2a88abd-1bec-4151-957e-d07dd2a80242","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f28e207f-7373-4e7c-be7d-4a9a962778d4","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f28e207f-7373-4e7c-be7d-4a9a962778d4","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f28e207f-7373-4e7c-be7d-4a9a962778d4","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f28e207f-7373-4e7c-be7d-4a9a962778d4"}
{"level":"error","ts":"2026-09-18T10:03:15.299Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"f28e207f-7373-4e7c-be7d-4a9a962778d4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.300Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"40cf0ac4-5e51-4b61-a262-64a4fb67e979","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.300Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"40cf0ac4-5e51-4b61-a262-64a4fb67e979","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.309Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"46cdf1cf-3950-4712-9bc5-65f95e86d2ab"}
{"level":"info","ts":"2026-09-18T10:03:15.309Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"46cdf1cf-3950-4712-9bc5-65f95e86d2ab"}
{"level":"info","ts":"2026-09-18T10:03:15.309Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"46cdf1cf-3950-4712-9bc5-65f95e86d2ab"}
{"level":"info","ts":"2026-09-18T10:03:15.309Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"46cdf1cf-3950-4712-9bc5-65f95e86d2ab","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:15.309Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"46cdf1cf-3950-4712-9bc5-65f95e86d2ab","error":"head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"error","ts":"2026-09-18T10:03:15.309Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"46cdf1cf-3950-4712-9bc5-65f95e86d2ab","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.310Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"46cdf1cf-3950-4712-9bc5-65f95e86d2ab","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"info","ts":"2026-09-18T10:03:15.310Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"46cdf1cf-3950-4712-9bc5-65f95e86d2ab","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.310Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"46cdf1cf-3950-4712-9bc5-65f95e86d2ab","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.310Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"46cdf1cf-3950-4712-9bc5-65f95e86d2ab","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.310Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"46cdf1cf-3950-4712-9bc5-65f95e86d2ab"}
{"level":"error","ts":"2026-09-18T10:03:15.310Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"46cdf1cf-3950-4712-9bc5-65f95e86d2ab","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.311Z","logger":"raycluster-resource","msg":"default","name":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.311Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-5qfl8","namespace":"test-ns-ltx56"}
{"level":"error","ts":"2026-09-18T10:03:15.314Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"7e5e46eb-3b63-40c8-a5b4-a45b60b5d4eb","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-5qfl8\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.314Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"7e5e46eb-3b63-40c8-a5b4-a45b60b5d4eb"}
{"level":"error","ts":"2026-09-18T10:03:15.314Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"7e5e46eb-3b63-40c8-a5b4-a45b60b5d4eb","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-5qfl8\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.314Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"e12c363c-bda1-47bf-aca3-9e5d404b9e76","namespacedName":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:15.314Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"e12c363c-bda1-47bf-aca3-9e5d404b9e76","mode":"IntegratedOAuth","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.314Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"e12c363c-bda1-47bf-aca3-9e5d404b9e76","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.314Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"e12c363c-bda1-47bf-aca3-9e5d404b9e76","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.321Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"359f5833-8492-4e6e-9167-81ea5be4c9d4","name":"long-running-5qfl8-head"}
{"level":"info","ts":"2026-09-18T10:03:15.366Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"a02cec3c-9128-4ad9-ace4-5b70415fbd7f"}
{"level":"info","ts":"2026-09-18T10:03:15.366Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"a02cec3c-9128-4ad9-ace4-5b70415fbd7f"}
{"level":"info","ts":"2026-09-18T10:03:15.366Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"a02cec3c-9128-4ad9-ace4-5b70415fbd7f"}
{"level":"info","ts":"2026-09-18T10:03:15.366Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"a02cec3c-9128-4ad9-ace4-5b70415fbd7f","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:15.366Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"a02cec3c-9128-4ad9-ace4-5b70415fbd7f","error":"head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"error","ts":"2026-09-18T10:03:15.366Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"a02cec3c-9128-4ad9-ace4-5b70415fbd7f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.366Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"a02cec3c-9128-4ad9-ace4-5b70415fbd7f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"info","ts":"2026-09-18T10:03:15.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"a02cec3c-9128-4ad9-ace4-5b70415fbd7f","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"a02cec3c-9128-4ad9-ace4-5b70415fbd7f","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"a02cec3c-9128-4ad9-ace4-5b70415fbd7f","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.366Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"a02cec3c-9128-4ad9-ace4-5b70415fbd7f"}
{"level":"error","ts":"2026-09-18T10:03:15.366Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"a02cec3c-9128-4ad9-ace4-5b70415fbd7f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.366Z","logger":"raycluster-resource","msg":"default","name":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.366Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-5qfl8","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:03:15.367Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"1b2bc0fa-de73-40a1-b3f3-7c62cc7e52ad","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:15.367Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"1b2bc0fa-de73-40a1-b3f3-7c62cc7e52ad","name":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.367Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"1b2bc0fa-de73-40a1-b3f3-7c62cc7e52ad","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:15.367Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"1b2bc0fa-de73-40a1-b3f3-7c62cc7e52ad"}
{"level":"info","ts":"2026-09-18T10:03:15.367Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"1b2bc0fa-de73-40a1-b3f3-7c62cc7e52ad","RayCluster":"long-running-5qfl8","State":""}
{"level":"info","ts":"2026-09-18T10:03:15.367Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"1b2bc0fa-de73-40a1-b3f3-7c62cc7e52ad","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:15.367Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"40cf0ac4-5e51-4b61-a262-64a4fb67e979","rayCluster":"long-running-5qfl8","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T10:03:15.367Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"40cf0ac4-5e51-4b61-a262-64a4fb67e979","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.377Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"359f5833-8492-4e6e-9167-81ea5be4c9d4","name":"long-running-5qfl8-workers"}
{"level":"info","ts":"2026-09-18T10:03:15.377Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"359f5833-8492-4e6e-9167-81ea5be4c9d4","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.377Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"48bec884-2c87-4a00-80ea-21fbae18aee6","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.396Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"dad09b83-4060-42a1-bc9e-66bf92d2e404","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:15.396Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"dad09b83-4060-42a1-bc9e-66bf92d2e404","name":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.396Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"dad09b83-4060-42a1-bc9e-66bf92d2e404","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:15.396Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"dad09b83-4060-42a1-bc9e-66bf92d2e404"}
{"level":"info","ts":"2026-09-18T10:03:15.396Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"dad09b83-4060-42a1-bc9e-66bf92d2e404","RayCluster":"long-running-5qfl8","State":""}
{"level":"info","ts":"2026-09-18T10:03:15.396Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"dad09b83-4060-42a1-bc9e-66bf92d2e404","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:15.407Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"40cf0ac4-5e51-4b61-a262-64a4fb67e979","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.417Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"e12c363c-bda1-47bf-aca3-9e5d404b9e76","name":"long-running-5qfl8-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:03:15.418Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"48bec884-2c87-4a00-80ea-21fbae18aee6","name":"long-running-5qfl8-head"}
{"level":"info","ts":"2026-09-18T10:03:15.430Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"e12c363c-bda1-47bf-aca3-9e5d404b9e76","name":"kuberay-gateway-access","namespace":"test-ns-ltx56","operation":"created"}
{"level":"info","ts":"2026-09-18T10:03:15.466Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"58404962-0e6c-49d4-9de8-15c1c9d5944a"}
{"level":"info","ts":"2026-09-18T10:03:15.466Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"58404962-0e6c-49d4-9de8-15c1c9d5944a"}
{"level":"info","ts":"2026-09-18T10:03:15.466Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"58404962-0e6c-49d4-9de8-15c1c9d5944a"}
{"level":"info","ts":"2026-09-18T10:03:15.466Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"58404962-0e6c-49d4-9de8-15c1c9d5944a","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:15.466Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"58404962-0e6c-49d4-9de8-15c1c9d5944a","error":"head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"error","ts":"2026-09-18T10:03:15.466Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"58404962-0e6c-49d4-9de8-15c1c9d5944a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.466Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"58404962-0e6c-49d4-9de8-15c1c9d5944a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"info","ts":"2026-09-18T10:03:15.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"58404962-0e6c-49d4-9de8-15c1c9d5944a","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"58404962-0e6c-49d4-9de8-15c1c9d5944a","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"58404962-0e6c-49d4-9de8-15c1c9d5944a","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.466Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"58404962-0e6c-49d4-9de8-15c1c9d5944a"}
{"level":"error","ts":"2026-09-18T10:03:15.466Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"58404962-0e6c-49d4-9de8-15c1c9d5944a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.466Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"40cf0ac4-5e51-4b61-a262-64a4fb67e979","rayCluster":"long-running-5qfl8","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:03:15.466Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"48bec884-2c87-4a00-80ea-21fbae18aee6","name":"long-running-5qfl8-workers"}
{"level":"info","ts":"2026-09-18T10:03:15.466Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"48bec884-2c87-4a00-80ea-21fbae18aee6","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.466Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"deac09e5-d5dd-4925-bca2-0a12e7d601d6","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.466Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"e12c363c-bda1-47bf-aca3-9e5d404b9e76","name":"test-ns-ltx56-long-running-5qfl8","operation":"created"}
{"level":"info","ts":"2026-09-18T10:03:15.486Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"e12c363c-bda1-47bf-aca3-9e5d404b9e76","name":"kube-rbac-proxy-config-long-running-5qfl8","operation":"created"}
{"level":"info","ts":"2026-09-18T10:03:15.491Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"40cf0ac4-5e51-4b61-a262-64a4fb67e979","rayCluster":"long-running-5qfl8","podIPs":[]}
{"level":"error","ts":"2026-09-18T10:03:15.491Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"40cf0ac4-5e51-4b61-a262-64a4fb67e979","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-5qfl8\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.491Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"40cf0ac4-5e51-4b61-a262-64a4fb67e979"}
{"level":"error","ts":"2026-09-18T10:03:15.491Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"40cf0ac4-5e51-4b61-a262-64a4fb67e979","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-5qfl8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.491Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"b6489edb-929d-4a6d-8b3e-30fc281b75bd","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.513Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"deac09e5-d5dd-4925-bca2-0a12e7d601d6","name":"long-running-5qfl8-head"}
{"level":"info","ts":"2026-09-18T10:03:15.513Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"e12c363c-bda1-47bf-aca3-9e5d404b9e76","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.513Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"58676b42-2299-4821-8221-df4a8186a25e","namespacedName":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:15.513Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"58676b42-2299-4821-8221-df4a8186a25e","mode":"IntegratedOAuth","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.513Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"58676b42-2299-4821-8221-df4a8186a25e","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.522Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"ce80d23c-edd8-4a6a-8e53-65b7103c4d27","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:15.522Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"ce80d23c-edd8-4a6a-8e53-65b7103c4d27","name":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.522Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"ce80d23c-edd8-4a6a-8e53-65b7103c4d27","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:15.522Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"ce80d23c-edd8-4a6a-8e53-65b7103c4d27"}
{"level":"info","ts":"2026-09-18T10:03:15.522Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"ce80d23c-edd8-4a6a-8e53-65b7103c4d27","RayCluster":"long-running-5qfl8","State":""}
{"level":"info","ts":"2026-09-18T10:03:15.522Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"ce80d23c-edd8-4a6a-8e53-65b7103c4d27","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:15.522Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"ce80d23c-edd8-4a6a-8e53-65b7103c4d27","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"error","ts":"2026-09-18T10:03:15.541Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"58676b42-2299-4821-8221-df4a8186a25e","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-5qfl8\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.541Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"58676b42-2299-4821-8221-df4a8186a25e"}
{"level":"error","ts":"2026-09-18T10:03:15.541Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"58676b42-2299-4821-8221-df4a8186a25e","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-5qfl8\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.546Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"ae0425c0-0ae8-440a-999c-b6ec1bb50f59","namespacedName":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:15.546Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"ae0425c0-0ae8-440a-999c-b6ec1bb50f59","mode":"IntegratedOAuth","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.546Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"ae0425c0-0ae8-440a-999c-b6ec1bb50f59","cluster":"long-running-5qfl8"}
{"level":"error","ts":"2026-09-18T10:03:15.566Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"b6489edb-929d-4a6d-8b3e-30fc281b75bd","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-5qfl8\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:03:15.566Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"b6489edb-929d-4a6d-8b3e-30fc281b75bd","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-5qfl8\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.566Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"b6489edb-929d-4a6d-8b3e-30fc281b75bd"}
{"level":"error","ts":"2026-09-18T10:03:15.566Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"b6489edb-929d-4a6d-8b3e-30fc281b75bd","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-5qfl8\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.566Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"95325eb4-2b41-490e-9233-86a8fddba257","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.566Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"95325eb4-2b41-490e-9233-86a8fddba257"}
{"level":"info","ts":"2026-09-18T10:03:15.566Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"a0f2dc1e-bf8f-4e02-8f79-f5e4e5ab4805","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:15.567Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"a0f2dc1e-bf8f-4e02-8f79-f5e4e5ab4805","name":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.567Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"a0f2dc1e-bf8f-4e02-8f79-f5e4e5ab4805","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:15.567Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"a0f2dc1e-bf8f-4e02-8f79-f5e4e5ab4805"}
{"level":"info","ts":"2026-09-18T10:03:15.567Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"a0f2dc1e-bf8f-4e02-8f79-f5e4e5ab4805","RayCluster":"long-running-5qfl8","State":""}
{"level":"info","ts":"2026-09-18T10:03:15.567Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"a0f2dc1e-bf8f-4e02-8f79-f5e4e5ab4805","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:15.575Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"ae0425c0-0ae8-440a-999c-b6ec1bb50f59","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.576Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"fa6af8ae-465c-449d-b433-92690cb31d84","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.576Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"fa6af8ae-465c-449d-b433-92690cb31d84"}
{"level":"info","ts":"2026-09-18T10:03:15.581Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"deac09e5-d5dd-4925-bca2-0a12e7d601d6","name":"long-running-5qfl8-workers"}
{"level":"info","ts":"2026-09-18T10:03:15.581Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"deac09e5-d5dd-4925-bca2-0a12e7d601d6","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.581Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"c97d3664-f9dd-4c8a-988e-c4a609ba631c","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.602Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"c97d3664-f9dd-4c8a-988e-c4a609ba631c","name":"long-running-5qfl8-head"}
{"level":"info","ts":"2026-09-18T10:03:15.626Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"be252583-3a01-4279-869d-937ca9ca2050"}
{"level":"info","ts":"2026-09-18T10:03:15.626Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"be252583-3a01-4279-869d-937ca9ca2050"}
{"level":"info","ts":"2026-09-18T10:03:15.626Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"be252583-3a01-4279-869d-937ca9ca2050"}
{"level":"info","ts":"2026-09-18T10:03:15.626Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"be252583-3a01-4279-869d-937ca9ca2050","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:15.626Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"be252583-3a01-4279-869d-937ca9ca2050","error":"head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"error","ts":"2026-09-18T10:03:15.626Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"be252583-3a01-4279-869d-937ca9ca2050","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.626Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"be252583-3a01-4279-869d-937ca9ca2050","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"info","ts":"2026-09-18T10:03:15.626Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"be252583-3a01-4279-869d-937ca9ca2050","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.626Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"be252583-3a01-4279-869d-937ca9ca2050","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"be252583-3a01-4279-869d-937ca9ca2050","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.627Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"be252583-3a01-4279-869d-937ca9ca2050"}
{"level":"error","ts":"2026-09-18T10:03:15.627Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"be252583-3a01-4279-869d-937ca9ca2050","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.634Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"c97d3664-f9dd-4c8a-988e-c4a609ba631c","name":"long-running-5qfl8-workers"}
{"level":"info","ts":"2026-09-18T10:03:15.634Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"c97d3664-f9dd-4c8a-988e-c4a609ba631c","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:15.948Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"67299578-0ca1-4c94-8b8b-a413cd70b3b1"}
{"level":"info","ts":"2026-09-18T10:03:15.948Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"67299578-0ca1-4c94-8b8b-a413cd70b3b1"}
{"level":"info","ts":"2026-09-18T10:03:15.948Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"67299578-0ca1-4c94-8b8b-a413cd70b3b1"}
{"level":"info","ts":"2026-09-18T10:03:15.948Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"67299578-0ca1-4c94-8b8b-a413cd70b3b1","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:15.948Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"67299578-0ca1-4c94-8b8b-a413cd70b3b1","error":"head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"error","ts":"2026-09-18T10:03:15.948Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"67299578-0ca1-4c94-8b8b-a413cd70b3b1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:15.948Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"67299578-0ca1-4c94-8b8b-a413cd70b3b1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"info","ts":"2026-09-18T10:03:15.948Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"67299578-0ca1-4c94-8b8b-a413cd70b3b1","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.948Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"67299578-0ca1-4c94-8b8b-a413cd70b3b1","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.948Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"67299578-0ca1-4c94-8b8b-a413cd70b3b1","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:15.948Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"67299578-0ca1-4c94-8b8b-a413cd70b3b1"}
{"level":"error","ts":"2026-09-18T10:03:15.948Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"67299578-0ca1-4c94-8b8b-a413cd70b3b1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:16.589Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1aed9e2c-1188-4e44-ae7e-31a26afcf061"}
{"level":"info","ts":"2026-09-18T10:03:16.589Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1aed9e2c-1188-4e44-ae7e-31a26afcf061"}
{"level":"info","ts":"2026-09-18T10:03:16.589Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1aed9e2c-1188-4e44-ae7e-31a26afcf061"}
{"level":"info","ts":"2026-09-18T10:03:16.589Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1aed9e2c-1188-4e44-ae7e-31a26afcf061","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:16.589Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1aed9e2c-1188-4e44-ae7e-31a26afcf061","error":"head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"error","ts":"2026-09-18T10:03:16.589Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1aed9e2c-1188-4e44-ae7e-31a26afcf061","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:16.589Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1aed9e2c-1188-4e44-ae7e-31a26afcf061","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found"}
{"level":"info","ts":"2026-09-18T10:03:16.589Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1aed9e2c-1188-4e44-ae7e-31a26afcf061","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:16.589Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1aed9e2c-1188-4e44-ae7e-31a26afcf061","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:16.589Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1aed9e2c-1188-4e44-ae7e-31a26afcf061","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:16.589Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1aed9e2c-1188-4e44-ae7e-31a26afcf061"}
{"level":"error","ts":"2026-09-18T10:03:16.589Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1aed9e2c-1188-4e44-ae7e-31a26afcf061","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5qfl8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:17.870Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7"}
{"level":"info","ts":"2026-09-18T10:03:17.870Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7"}
{"level":"info","ts":"2026-09-18T10:03:17.870Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7"}
{"level":"info","ts":"2026-09-18T10:03:17.870Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:17.870Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7"}
{"level":"info","ts":"2026-09-18T10:03:17.870Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:03:17.870Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7"}
{"level":"info","ts":"2026-09-18T10:03:17.870Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","mode":"IntegratedOAuth","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:17.870Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","cluster":"long-running-5qfl8","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:03:17.870Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","cluster":"long-running-5qfl8","authType":"IntegratedOAuth","serviceAccount":"long-running-5qfl8-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T10:03:17.870Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-5qfl8","ray.io/group":"headgroup","ray.io/identifier":"long-running-5qfl8-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:03:17.870Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T10:03:17.871Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:03:17.871Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:03:17.871Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","enabled":true}
{"level":"info","ts":"2026-09-18T10:03:17.902Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","name":"long-running-5qfl8-head-kflfc"}
{"level":"info","ts":"2026-09-18T10:03:17.902Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:17.902Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:17.902Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:03:17.902Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:03:17.902Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:17.902Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:03:17.902Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7"}
{"level":"info","ts":"2026-09-18T10:03:17.902Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","nodeType":"worker","rayStartParams":{"address":"long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:03:17.902Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","rayStartCmd":"ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:03:17.902Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:03:17.902Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","enabled":true}
{"level":"info","ts":"2026-09-18T10:03:17.945Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","name":"long-running-5qfl8-small-group-worker-tbqtj"}
{"level":"info","ts":"2026-09-18T10:03:17.945Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:17.945Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"997b6014-c23d-4a19-a792-6eb075bdcbc7","name":"long-running-5qfl8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:03:17.973Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"22783280-0b94-4809-abc2-3ef3a0fbdf87","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:17.973Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"05b20a04-f477-4fa2-9675-1cc23e276a70","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:17.973Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"05b20a04-f477-4fa2-9675-1cc23e276a70","name":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:17.973Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"05b20a04-f477-4fa2-9675-1cc23e276a70","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:17.973Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"22783280-0b94-4809-abc2-3ef3a0fbdf87","rayCluster":"long-running-5qfl8","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:03:17.973Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"091157a9-3f0b-44c2-9838-32fce186b989","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:17.973Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"05b20a04-f477-4fa2-9675-1cc23e276a70"}
{"level":"info","ts":"2026-09-18T10:03:17.973Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"05b20a04-f477-4fa2-9675-1cc23e276a70","RayCluster":"long-running-5qfl8","State":""}
{"level":"info","ts":"2026-09-18T10:03:17.973Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"05b20a04-f477-4fa2-9675-1cc23e276a70","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:17.973Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"05b20a04-f477-4fa2-9675-1cc23e276a70","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:03:17.978Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2601258e-f8b0-4a7b-b583-fd9b4ea4a7ad"}
{"level":"info","ts":"2026-09-18T10:03:17.978Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2601258e-f8b0-4a7b-b583-fd9b4ea4a7ad"}
{"level":"info","ts":"2026-09-18T10:03:17.978Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2601258e-f8b0-4a7b-b583-fd9b4ea4a7ad"}
{"level":"info","ts":"2026-09-18T10:03:17.978Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2601258e-f8b0-4a7b-b583-fd9b4ea4a7ad","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:17.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2601258e-f8b0-4a7b-b583-fd9b4ea4a7ad","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:17.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2601258e-f8b0-4a7b-b583-fd9b4ea4a7ad","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:17.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2601258e-f8b0-4a7b-b583-fd9b4ea4a7ad","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:17.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2601258e-f8b0-4a7b-b583-fd9b4ea4a7ad","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:17.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2601258e-f8b0-4a7b-b583-fd9b4ea4a7ad","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:17.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2601258e-f8b0-4a7b-b583-fd9b4ea4a7ad","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:17.978Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2601258e-f8b0-4a7b-b583-fd9b4ea4a7ad","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:17.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2601258e-f8b0-4a7b-b583-fd9b4ea4a7ad","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:17.978Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2601258e-f8b0-4a7b-b583-fd9b4ea4a7ad","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:17.978Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2601258e-f8b0-4a7b-b583-fd9b4ea4a7ad","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:17.978Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2601258e-f8b0-4a7b-b583-fd9b4ea4a7ad","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:17.978Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2601258e-f8b0-4a7b-b583-fd9b4ea4a7ad","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:17.993Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"98310825-7859-4e35-80c2-a00f50c007b4"}
{"level":"info","ts":"2026-09-18T10:03:17.993Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"98310825-7859-4e35-80c2-a00f50c007b4"}
{"level":"info","ts":"2026-09-18T10:03:17.993Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"98310825-7859-4e35-80c2-a00f50c007b4"}
{"level":"info","ts":"2026-09-18T10:03:17.993Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"98310825-7859-4e35-80c2-a00f50c007b4","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:17.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"98310825-7859-4e35-80c2-a00f50c007b4","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:17.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"98310825-7859-4e35-80c2-a00f50c007b4","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:17.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"98310825-7859-4e35-80c2-a00f50c007b4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:17.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"98310825-7859-4e35-80c2-a00f50c007b4","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:17.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"98310825-7859-4e35-80c2-a00f50c007b4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:17.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"98310825-7859-4e35-80c2-a00f50c007b4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:17.993Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"98310825-7859-4e35-80c2-a00f50c007b4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:17.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"98310825-7859-4e35-80c2-a00f50c007b4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:17.993Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"98310825-7859-4e35-80c2-a00f50c007b4","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:17.993Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"98310825-7859-4e35-80c2-a00f50c007b4","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:17.993Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"98310825-7859-4e35-80c2-a00f50c007b4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:17.993Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"98310825-7859-4e35-80c2-a00f50c007b4","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:18.002Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"a0a70ecd-a06a-4e2c-8d14-4bb84b4d7759","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:18.068Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"a0a70ecd-a06a-4e2c-8d14-4bb84b4d7759","name":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:18.068Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"a0a70ecd-a06a-4e2c-8d14-4bb84b4d7759","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:18.069Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"a0a70ecd-a06a-4e2c-8d14-4bb84b4d7759"}
{"level":"info","ts":"2026-09-18T10:03:18.069Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"a0a70ecd-a06a-4e2c-8d14-4bb84b4d7759","RayCluster":"long-running-5qfl8","State":""}
{"level":"info","ts":"2026-09-18T10:03:18.069Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"a0a70ecd-a06a-4e2c-8d14-4bb84b4d7759","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:18.069Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310153e8-ebaf-454c-943d-b10a547d0305"}
{"level":"info","ts":"2026-09-18T10:03:18.069Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310153e8-ebaf-454c-943d-b10a547d0305"}
{"level":"info","ts":"2026-09-18T10:03:18.069Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310153e8-ebaf-454c-943d-b10a547d0305"}
{"level":"info","ts":"2026-09-18T10:03:18.069Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310153e8-ebaf-454c-943d-b10a547d0305","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:18.069Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310153e8-ebaf-454c-943d-b10a547d0305","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:18.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310153e8-ebaf-454c-943d-b10a547d0305","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:18.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310153e8-ebaf-454c-943d-b10a547d0305","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:18.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310153e8-ebaf-454c-943d-b10a547d0305","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:18.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310153e8-ebaf-454c-943d-b10a547d0305","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:18.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310153e8-ebaf-454c-943d-b10a547d0305","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:18.070Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310153e8-ebaf-454c-943d-b10a547d0305","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:18.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310153e8-ebaf-454c-943d-b10a547d0305","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:18.070Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310153e8-ebaf-454c-943d-b10a547d0305","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:18.070Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310153e8-ebaf-454c-943d-b10a547d0305","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:18.070Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310153e8-ebaf-454c-943d-b10a547d0305","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:18.070Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310153e8-ebaf-454c-943d-b10a547d0305","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:18.070Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"091157a9-3f0b-44c2-9838-32fce186b989","name":"long-running-5qfl8-head"}
{"level":"info","ts":"2026-09-18T10:03:18.097Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"091157a9-3f0b-44c2-9838-32fce186b989","name":"long-running-5qfl8-workers"}
{"level":"info","ts":"2026-09-18T10:03:18.097Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"091157a9-3f0b-44c2-9838-32fce186b989","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:18.238Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"8b5d7cdf-de31-41b3-8691-453ea8036907","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:18.238Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"8b5d7cdf-de31-41b3-8691-453ea8036907","name":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:18.238Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"8b5d7cdf-de31-41b3-8691-453ea8036907","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:18.238Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"8b5d7cdf-de31-41b3-8691-453ea8036907"}
{"level":"info","ts":"2026-09-18T10:03:18.238Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"8b5d7cdf-de31-41b3-8691-453ea8036907","RayCluster":"long-running-5qfl8","State":""}
{"level":"info","ts":"2026-09-18T10:03:18.238Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"8b5d7cdf-de31-41b3-8691-453ea8036907","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:18.578Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1a1b3248-29c2-4c55-b663-b4896d95cb85"}
{"level":"info","ts":"2026-09-18T10:03:18.578Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1a1b3248-29c2-4c55-b663-b4896d95cb85"}
{"level":"info","ts":"2026-09-18T10:03:18.578Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1a1b3248-29c2-4c55-b663-b4896d95cb85"}
{"level":"info","ts":"2026-09-18T10:03:18.578Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1a1b3248-29c2-4c55-b663-b4896d95cb85","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:18.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1a1b3248-29c2-4c55-b663-b4896d95cb85","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:18.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1a1b3248-29c2-4c55-b663-b4896d95cb85","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:18.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1a1b3248-29c2-4c55-b663-b4896d95cb85","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:18.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1a1b3248-29c2-4c55-b663-b4896d95cb85","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:18.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1a1b3248-29c2-4c55-b663-b4896d95cb85","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:18.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1a1b3248-29c2-4c55-b663-b4896d95cb85","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:18.578Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1a1b3248-29c2-4c55-b663-b4896d95cb85","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:18.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1a1b3248-29c2-4c55-b663-b4896d95cb85","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:18.578Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1a1b3248-29c2-4c55-b663-b4896d95cb85","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:18.578Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1a1b3248-29c2-4c55-b663-b4896d95cb85","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:18.578Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1a1b3248-29c2-4c55-b663-b4896d95cb85","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:18.578Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1a1b3248-29c2-4c55-b663-b4896d95cb85","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:19.246Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4edc1347-3383-4207-87f5-e54eb5a7e268"}
{"level":"info","ts":"2026-09-18T10:03:19.246Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4edc1347-3383-4207-87f5-e54eb5a7e268"}
{"level":"info","ts":"2026-09-18T10:03:19.246Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4edc1347-3383-4207-87f5-e54eb5a7e268"}
{"level":"info","ts":"2026-09-18T10:03:19.246Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4edc1347-3383-4207-87f5-e54eb5a7e268","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:19.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4edc1347-3383-4207-87f5-e54eb5a7e268","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:19.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4edc1347-3383-4207-87f5-e54eb5a7e268","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:19.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4edc1347-3383-4207-87f5-e54eb5a7e268","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:19.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4edc1347-3383-4207-87f5-e54eb5a7e268","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:19.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4edc1347-3383-4207-87f5-e54eb5a7e268","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:19.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4edc1347-3383-4207-87f5-e54eb5a7e268","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:19.247Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4edc1347-3383-4207-87f5-e54eb5a7e268","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:19.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4edc1347-3383-4207-87f5-e54eb5a7e268","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:19.247Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4edc1347-3383-4207-87f5-e54eb5a7e268","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:19.247Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4edc1347-3383-4207-87f5-e54eb5a7e268","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:19.247Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4edc1347-3383-4207-87f5-e54eb5a7e268","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:19.247Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4edc1347-3383-4207-87f5-e54eb5a7e268","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:19.978Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"482e55b4-0de8-4920-9594-a3aa0a5a8512"}
{"level":"info","ts":"2026-09-18T10:03:19.978Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"482e55b4-0de8-4920-9594-a3aa0a5a8512"}
{"level":"info","ts":"2026-09-18T10:03:19.978Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"482e55b4-0de8-4920-9594-a3aa0a5a8512"}
{"level":"info","ts":"2026-09-18T10:03:19.978Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"482e55b4-0de8-4920-9594-a3aa0a5a8512","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:19.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"482e55b4-0de8-4920-9594-a3aa0a5a8512","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:19.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"482e55b4-0de8-4920-9594-a3aa0a5a8512","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:19.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"482e55b4-0de8-4920-9594-a3aa0a5a8512","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:19.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"482e55b4-0de8-4920-9594-a3aa0a5a8512","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:19.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"482e55b4-0de8-4920-9594-a3aa0a5a8512","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:19.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"482e55b4-0de8-4920-9594-a3aa0a5a8512","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:19.978Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"482e55b4-0de8-4920-9594-a3aa0a5a8512","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:19.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"482e55b4-0de8-4920-9594-a3aa0a5a8512","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:19.979Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"482e55b4-0de8-4920-9594-a3aa0a5a8512","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:19.979Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"482e55b4-0de8-4920-9594-a3aa0a5a8512","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:19.979Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"482e55b4-0de8-4920-9594-a3aa0a5a8512","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:19.979Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"482e55b4-0de8-4920-9594-a3aa0a5a8512","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:20.315Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ab94a99-c25b-4afe-83a5-b346ba9e9376"}
{"level":"info","ts":"2026-09-18T10:03:20.315Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ab94a99-c25b-4afe-83a5-b346ba9e9376"}
{"level":"info","ts":"2026-09-18T10:03:20.315Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ab94a99-c25b-4afe-83a5-b346ba9e9376"}
{"level":"info","ts":"2026-09-18T10:03:20.315Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ab94a99-c25b-4afe-83a5-b346ba9e9376","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:20.315Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ab94a99-c25b-4afe-83a5-b346ba9e9376","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:20.315Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ab94a99-c25b-4afe-83a5-b346ba9e9376","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:20.315Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ab94a99-c25b-4afe-83a5-b346ba9e9376","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:20.315Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ab94a99-c25b-4afe-83a5-b346ba9e9376","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:20.315Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ab94a99-c25b-4afe-83a5-b346ba9e9376","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:20.315Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ab94a99-c25b-4afe-83a5-b346ba9e9376","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:20.315Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ab94a99-c25b-4afe-83a5-b346ba9e9376","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:20.315Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ab94a99-c25b-4afe-83a5-b346ba9e9376","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:20.315Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ab94a99-c25b-4afe-83a5-b346ba9e9376","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:20.315Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ab94a99-c25b-4afe-83a5-b346ba9e9376","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:20.315Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ab94a99-c25b-4afe-83a5-b346ba9e9376","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:20.315Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ab94a99-c25b-4afe-83a5-b346ba9e9376","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:20.645Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fb318d54-e414-4f83-8918-90646763df6f"}
{"level":"info","ts":"2026-09-18T10:03:20.645Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fb318d54-e414-4f83-8918-90646763df6f"}
{"level":"info","ts":"2026-09-18T10:03:20.645Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fb318d54-e414-4f83-8918-90646763df6f"}
{"level":"info","ts":"2026-09-18T10:03:20.645Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fb318d54-e414-4f83-8918-90646763df6f","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:20.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fb318d54-e414-4f83-8918-90646763df6f","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:20.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fb318d54-e414-4f83-8918-90646763df6f","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:20.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fb318d54-e414-4f83-8918-90646763df6f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:20.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fb318d54-e414-4f83-8918-90646763df6f","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:20.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fb318d54-e414-4f83-8918-90646763df6f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:20.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fb318d54-e414-4f83-8918-90646763df6f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:20.645Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fb318d54-e414-4f83-8918-90646763df6f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:20.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fb318d54-e414-4f83-8918-90646763df6f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:20.645Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fb318d54-e414-4f83-8918-90646763df6f","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:20.645Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fb318d54-e414-4f83-8918-90646763df6f","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:20.645Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fb318d54-e414-4f83-8918-90646763df6f","name":"long-running-5qfl8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.42","serviceIP":"10.133.0.42","podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:03:20.661Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a6caf8a3-f7b9-492c-b5dd-02f36c301cdb","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:20.661Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"0bc44e9e-b6c1-4078-aee8-edb1819e652a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:20.661Z","logger":"controllers.RayJob","msg":"The RayJob has passed the activeDeadlineSeconds. Transition the status to `Failed`.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"0bc44e9e-b6c1-4078-aee8-edb1819e652a","StartTime":"2026-09-18 10:03:15 +0000 UTC","ActiveDeadlineSeconds":5}
{"level":"info","ts":"2026-09-18T10:03:20.661Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"0bc44e9e-b6c1-4078-aee8-edb1819e652a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-rxs4j","rayClusterName":"long-running-5qfl8","jobDeploymentStatus":"Failed","reason":"DeadlineExceeded","message":"The RayJob has passed the activeDeadlineSeconds. StartTime: 2026-09-18 10:03:15 +0000 UTC. ActiveDeadlineSeconds: 5","startTime":"2026-09-18T10:03:15Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:20.661Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"0bc44e9e-b6c1-4078-aee8-edb1819e652a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-09-18T10:03:20.662Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"129fdbae-6909-4f81-8bfd-89d6233f9a5b","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:20.662Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"129fdbae-6909-4f81-8bfd-89d6233f9a5b","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:03:20.662Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"129fdbae-6909-4f81-8bfd-89d6233f9a5b","pod":"long-running-5qfl8-small-group-worker-tbqtj","ip":"10.132.0.47"}
{"level":"info","ts":"2026-09-18T10:03:20.681Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"129fdbae-6909-4f81-8bfd-89d6233f9a5b","certificate":"ray-head-cert-long-running-5qfl8","dnsNames":["long-running-5qfl8-head-svc","localhost","long-running-5qfl8-head-svc.test-ns-ltx56.svc","long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.42","10.132.0.47","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:03:20.693Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a6caf8a3-f7b9-492c-b5dd-02f36c301cdb","name":"long-running-5qfl8-head"}
{"level":"info","ts":"2026-09-18T10:03:20.696Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"5530d505-834a-42e9-a1f9-b27a546b95fd","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:20.696Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"5530d505-834a-42e9-a1f9-b27a546b95fd","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-09-18 10:03:20 +0000 UTC","Now":"2026-09-18T10:03:20.696Z","ShutdownTime":"2026-09-18T10:13:20.000Z"}
{"level":"info","ts":"2026-09-18T10:03:20.696Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"5530d505-834a-42e9-a1f9-b27a546b95fd","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-09-18T10:13:20.000Z","requeueAfter":601.303500853}
{"level":"info","ts":"2026-09-18T10:03:20.703Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"129fdbae-6909-4f81-8bfd-89d6233f9a5b","certificate":"ray-worker-cert-long-running-5qfl8","dnsNames":["long-running-5qfl8-worker-svc","localhost","long-running-5qfl8-small-group","long-running-5qfl8-worker-svc.test-ns-ltx56.svc","long-running-5qfl8-worker-svc.test-ns-ltx56.svc.cluster.local","*.long-running-5qfl8-worker-svc.test-ns-ltx56.svc","*.long-running-5qfl8-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.42","10.132.0.47","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:03:20.703Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"129fdbae-6909-4f81-8bfd-89d6233f9a5b","rayCluster":"long-running-5qfl8","podIPs":["10.133.0.42","10.132.0.47"]}
{"level":"info","ts":"2026-09-18T10:03:20.722Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a6caf8a3-f7b9-492c-b5dd-02f36c301cdb","name":"long-running-5qfl8-workers"}
{"level":"info","ts":"2026-09-18T10:03:20.722Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a6caf8a3-f7b9-492c-b5dd-02f36c301cdb","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:21.239Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"664b904d-59b5-4c6e-95e4-4680d2e2add3","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:21.239Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"664b904d-59b5-4c6e-95e4-4680d2e2add3","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-09-18 10:03:20 +0000 UTC","Now":"2026-09-18T10:03:21.239Z","ShutdownTime":"2026-09-18T10:13:20.000Z"}
{"level":"info","ts":"2026-09-18T10:03:21.239Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"664b904d-59b5-4c6e-95e4-4680d2e2add3","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-09-18T10:13:20.000Z","requeueAfter":600.760750733}
{"level":"info","ts":"2026-09-18T10:03:21.259Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b0385bbc-b820-421d-9f59-7dec011ad59d","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:21.259Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b0385bbc-b820-421d-9f59-7dec011ad59d","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T10:03:21.278Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b0385bbc-b820-421d-9f59-7dec011ad59d"}
{"level":"info","ts":"2026-09-18T10:03:21.278Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b0385bbc-b820-421d-9f59-7dec011ad59d","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T10:03:21.278Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b0385bbc-b820-421d-9f59-7dec011ad59d","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:03:21.278Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b0385bbc-b820-421d-9f59-7dec011ad59d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:03:21.304Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9c248bc1-434e-49ef-8106-800c89a66060","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:21.304Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9c248bc1-434e-49ef-8106-800c89a66060","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.304Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9c248bc1-434e-49ef-8106-800c89a66060","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:21.304Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9c248bc1-434e-49ef-8106-800c89a66060","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:21.312Z","logger":"raycluster-resource","msg":"default","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.312Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9c248bc1-434e-49ef-8106-800c89a66060","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6169f8e2-1ff7-4541-8409-717cdfbcbc44"}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6169f8e2-1ff7-4541-8409-717cdfbcbc44"}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6169f8e2-1ff7-4541-8409-717cdfbcbc44"}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9c248bc1-434e-49ef-8106-800c89a66060"}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9c248bc1-434e-49ef-8106-800c89a66060","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9c248bc1-434e-49ef-8106-800c89a66060","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef7e056d-1b20-46da-8f0d-67c60be588e1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef7e056d-1b20-46da-8f0d-67c60be588e1","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef7e056d-1b20-46da-8f0d-67c60be588e1","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef7e056d-1b20-46da-8f0d-67c60be588e1"}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef7e056d-1b20-46da-8f0d-67c60be588e1","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef7e056d-1b20-46da-8f0d-67c60be588e1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"928fce9a-363c-49dc-9767-9a16b8c5e170","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"53169bac-0f7e-42d0-b5e3-540212b8191b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"53169bac-0f7e-42d0-b5e3-540212b8191b","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"53169bac-0f7e-42d0-b5e3-540212b8191b","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"53169bac-0f7e-42d0-b5e3-540212b8191b"}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"53169bac-0f7e-42d0-b5e3-540212b8191b","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"53169bac-0f7e-42d0-b5e3-540212b8191b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"601e44ae-9bb4-4935-98b8-4aca597139f0","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"601e44ae-9bb4-4935-98b8-4aca597139f0","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f375fe96-cd64-4f47-ac8c-fb2908586f2e","namespacedName":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f375fe96-cd64-4f47-ac8c-fb2908586f2e","mode":"IntegratedOAuth","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.329Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f375fe96-cd64-4f47-ac8c-fb2908586f2e","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.330Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f375fe96-cd64-4f47-ac8c-fb2908586f2e","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.337Z","logger":"raycluster-resource","msg":"default","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.337Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:03:21.339Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6169f8e2-1ff7-4541-8409-717cdfbcbc44","name":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:21.339Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6169f8e2-1ff7-4541-8409-717cdfbcbc44","error":"head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"error","ts":"2026-09-18T10:03:21.339Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6169f8e2-1ff7-4541-8409-717cdfbcbc44","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.339Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6169f8e2-1ff7-4541-8409-717cdfbcbc44","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"info","ts":"2026-09-18T10:03:21.339Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6169f8e2-1ff7-4541-8409-717cdfbcbc44","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.339Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6169f8e2-1ff7-4541-8409-717cdfbcbc44","serviceSelectors":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/identifier":"delete-head-after-submit-r4xnr-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.339Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6169f8e2-1ff7-4541-8409-717cdfbcbc44","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.339Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6169f8e2-1ff7-4541-8409-717cdfbcbc44","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.339Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6169f8e2-1ff7-4541-8409-717cdfbcbc44","name":"delete-head-after-submit-r4xnr","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:21Z","head":{"serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:03:21.348Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"928fce9a-363c-49dc-9767-9a16b8c5e170","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.348Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"394f56d7-6bf8-4a7b-812f-20b6a83fb5ec","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.352Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"601e44ae-9bb4-4935-98b8-4aca597139f0","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.352Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"601e44ae-9bb4-4935-98b8-4aca597139f0","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.353Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a527035b-c329-4805-af1e-90bdd89aef31","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:21.353Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a527035b-c329-4805-af1e-90bdd89aef31","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.353Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a527035b-c329-4805-af1e-90bdd89aef31","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:21.353Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a527035b-c329-4805-af1e-90bdd89aef31"}
{"level":"info","ts":"2026-09-18T10:03:21.353Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a527035b-c329-4805-af1e-90bdd89aef31","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:21.353Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a527035b-c329-4805-af1e-90bdd89aef31","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:03:21.359Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6169f8e2-1ff7-4541-8409-717cdfbcbc44","name":"delete-head-after-submit-r4xnr","error":"Operation cannot be fulfilled on rayclusters.ray.io \"delete-head-after-submit-r4xnr\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr"}}
{"level":"info","ts":"2026-09-18T10:03:21.359Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6169f8e2-1ff7-4541-8409-717cdfbcbc44"}
{"level":"error","ts":"2026-09-18T10:03:21.359Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6169f8e2-1ff7-4541-8409-717cdfbcbc44","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.359Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6121579d-38b0-4a9c-9372-3777af6d48c2"}
{"level":"info","ts":"2026-09-18T10:03:21.359Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6121579d-38b0-4a9c-9372-3777af6d48c2"}
{"level":"info","ts":"2026-09-18T10:03:21.359Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6121579d-38b0-4a9c-9372-3777af6d48c2"}
{"level":"info","ts":"2026-09-18T10:03:21.359Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6121579d-38b0-4a9c-9372-3777af6d48c2","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:21.359Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6121579d-38b0-4a9c-9372-3777af6d48c2","error":"head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"error","ts":"2026-09-18T10:03:21.359Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6121579d-38b0-4a9c-9372-3777af6d48c2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.359Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6121579d-38b0-4a9c-9372-3777af6d48c2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"info","ts":"2026-09-18T10:03:21.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6121579d-38b0-4a9c-9372-3777af6d48c2","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6121579d-38b0-4a9c-9372-3777af6d48c2","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6121579d-38b0-4a9c-9372-3777af6d48c2","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.366Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6121579d-38b0-4a9c-9372-3777af6d48c2","name":"delete-head-after-submit-r4xnr","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:03:21.366Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f375fe96-cd64-4f47-ac8c-fb2908586f2e","name":"delete-head-after-submit-r4xnr-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:03:21.369Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"601e44ae-9bb4-4935-98b8-4aca597139f0","rayCluster":"delete-head-after-submit-r4xnr","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T10:03:21.369Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"601e44ae-9bb4-4935-98b8-4aca597139f0","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.374Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"394f56d7-6bf8-4a7b-812f-20b6a83fb5ec","name":"delete-head-after-submit-r4xnr-head"}
{"level":"info","ts":"2026-09-18T10:03:21.381Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f375fe96-cd64-4f47-ac8c-fb2908586f2e","name":"test-ns-ltx56-delete-head-after-submit-r4xnr","operation":"created"}
{"level":"info","ts":"2026-09-18T10:03:21.383Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6121579d-38b0-4a9c-9372-3777af6d48c2"}
{"level":"error","ts":"2026-09-18T10:03:21.383Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6121579d-38b0-4a9c-9372-3777af6d48c2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.383Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"a53b91a3-9050-43bd-a2ac-ebebc396fa55"}
{"level":"info","ts":"2026-09-18T10:03:21.383Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"a53b91a3-9050-43bd-a2ac-ebebc396fa55"}
{"level":"info","ts":"2026-09-18T10:03:21.383Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"a53b91a3-9050-43bd-a2ac-ebebc396fa55"}
{"level":"info","ts":"2026-09-18T10:03:21.383Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"a53b91a3-9050-43bd-a2ac-ebebc396fa55","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:21.383Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"a53b91a3-9050-43bd-a2ac-ebebc396fa55","error":"head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"error","ts":"2026-09-18T10:03:21.383Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"a53b91a3-9050-43bd-a2ac-ebebc396fa55","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.383Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"a53b91a3-9050-43bd-a2ac-ebebc396fa55","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"info","ts":"2026-09-18T10:03:21.383Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"a53b91a3-9050-43bd-a2ac-ebebc396fa55","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.383Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"a53b91a3-9050-43bd-a2ac-ebebc396fa55","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.383Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"a53b91a3-9050-43bd-a2ac-ebebc396fa55","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.383Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"a53b91a3-9050-43bd-a2ac-ebebc396fa55","name":"delete-head-after-submit-r4xnr","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:03:21.384Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a26feb0a-c96c-4710-ac58-8a92f71b7566","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:21.384Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a26feb0a-c96c-4710-ac58-8a92f71b7566","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.384Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a26feb0a-c96c-4710-ac58-8a92f71b7566","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:21.384Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a26feb0a-c96c-4710-ac58-8a92f71b7566"}
{"level":"info","ts":"2026-09-18T10:03:21.384Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a26feb0a-c96c-4710-ac58-8a92f71b7566","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:21.384Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a26feb0a-c96c-4710-ac58-8a92f71b7566","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:21.385Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a26feb0a-c96c-4710-ac58-8a92f71b7566","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:03:21.388Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"601e44ae-9bb4-4935-98b8-4aca597139f0","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.391Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f375fe96-cd64-4f47-ac8c-fb2908586f2e","name":"kube-rbac-proxy-config-delete-head-after-submit-r4xnr","operation":"created"}
{"level":"info","ts":"2026-09-18T10:03:21.397Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"a53b91a3-9050-43bd-a2ac-ebebc396fa55","name":"delete-head-after-submit-r4xnr","error":"Operation cannot be fulfilled on rayclusters.ray.io \"delete-head-after-submit-r4xnr\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr"}}
{"level":"info","ts":"2026-09-18T10:03:21.397Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"a53b91a3-9050-43bd-a2ac-ebebc396fa55"}
{"level":"error","ts":"2026-09-18T10:03:21.397Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"a53b91a3-9050-43bd-a2ac-ebebc396fa55","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.397Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"220fa100-5796-4865-bd00-09aa7c3e9192"}
{"level":"info","ts":"2026-09-18T10:03:21.397Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"220fa100-5796-4865-bd00-09aa7c3e9192"}
{"level":"info","ts":"2026-09-18T10:03:21.397Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"220fa100-5796-4865-bd00-09aa7c3e9192"}
{"level":"info","ts":"2026-09-18T10:03:21.397Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"220fa100-5796-4865-bd00-09aa7c3e9192","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:21.397Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"220fa100-5796-4865-bd00-09aa7c3e9192","error":"head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"error","ts":"2026-09-18T10:03:21.397Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"220fa100-5796-4865-bd00-09aa7c3e9192","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.397Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"220fa100-5796-4865-bd00-09aa7c3e9192","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"info","ts":"2026-09-18T10:03:21.397Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"220fa100-5796-4865-bd00-09aa7c3e9192","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.397Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"220fa100-5796-4865-bd00-09aa7c3e9192","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.397Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"220fa100-5796-4865-bd00-09aa7c3e9192","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.397Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"220fa100-5796-4865-bd00-09aa7c3e9192"}
{"level":"error","ts":"2026-09-18T10:03:21.397Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"220fa100-5796-4865-bd00-09aa7c3e9192","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"1297bccb-d5a4-4586-a3a0-b117fab2aa0e"}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"1297bccb-d5a4-4586-a3a0-b117fab2aa0e"}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"1297bccb-d5a4-4586-a3a0-b117fab2aa0e"}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"1297bccb-d5a4-4586-a3a0-b117fab2aa0e","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"1297bccb-d5a4-4586-a3a0-b117fab2aa0e","error":"head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"error","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"1297bccb-d5a4-4586-a3a0-b117fab2aa0e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"1297bccb-d5a4-4586-a3a0-b117fab2aa0e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"1297bccb-d5a4-4586-a3a0-b117fab2aa0e","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"1297bccb-d5a4-4586-a3a0-b117fab2aa0e","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"1297bccb-d5a4-4586-a3a0-b117fab2aa0e","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"1297bccb-d5a4-4586-a3a0-b117fab2aa0e"}
{"level":"error","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"1297bccb-d5a4-4586-a3a0-b117fab2aa0e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"394f56d7-6bf8-4a7b-812f-20b6a83fb5ec","name":"delete-head-after-submit-r4xnr-workers"}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"394f56d7-6bf8-4a7b-812f-20b6a83fb5ec","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"ddd8a2a2-c167-49d8-84e5-74ac3e49aa0d","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d6fe725f-a481-43bb-bd40-4b85e6249966","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d6fe725f-a481-43bb-bd40-4b85e6249966","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d6fe725f-a481-43bb-bd40-4b85e6249966","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d6fe725f-a481-43bb-bd40-4b85e6249966"}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d6fe725f-a481-43bb-bd40-4b85e6249966","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:21.466Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d6fe725f-a481-43bb-bd40-4b85e6249966","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:21.467Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"601e44ae-9bb4-4935-98b8-4aca597139f0","rayCluster":"delete-head-after-submit-r4xnr","podIPs":[]}
{"level":"error","ts":"2026-09-18T10:03:21.467Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f375fe96-cd64-4f47-ac8c-fb2908586f2e","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"delete-head-after-submit-r4xnr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.467Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f375fe96-cd64-4f47-ac8c-fb2908586f2e"}
{"level":"error","ts":"2026-09-18T10:03:21.467Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f375fe96-cd64-4f47-ac8c-fb2908586f2e","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"delete-head-after-submit-r4xnr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.467Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f0e254f0-977f-455a-b0ba-7672192e49bb","namespacedName":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:21.467Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f0e254f0-977f-455a-b0ba-7672192e49bb","mode":"IntegratedOAuth","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.467Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f0e254f0-977f-455a-b0ba-7672192e49bb","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.489Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f0e254f0-977f-455a-b0ba-7672192e49bb","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.489Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"73385ced-3b2c-4ff9-ac37-a9570e109812","namespacedName":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:21.489Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"73385ced-3b2c-4ff9-ac37-a9570e109812","mode":"IntegratedOAuth","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.489Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"73385ced-3b2c-4ff9-ac37-a9570e109812","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.490Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"381fbd7e-a1c0-4b10-947c-abbd3570930a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:21.490Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"381fbd7e-a1c0-4b10-947c-abbd3570930a","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.490Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"381fbd7e-a1c0-4b10-947c-abbd3570930a","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:21.490Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"381fbd7e-a1c0-4b10-947c-abbd3570930a"}
{"level":"info","ts":"2026-09-18T10:03:21.490Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"381fbd7e-a1c0-4b10-947c-abbd3570930a","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:21.490Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"381fbd7e-a1c0-4b10-947c-abbd3570930a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:21.490Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"381fbd7e-a1c0-4b10-947c-abbd3570930a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:03:21.498Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"601e44ae-9bb4-4935-98b8-4aca597139f0","rayCluster":"delete-head-after-submit-r4xnr","podIPs":[]}
{"level":"error","ts":"2026-09-18T10:03:21.498Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"601e44ae-9bb4-4935-98b8-4aca597139f0","error":"Certificate.cert-manager.io \"ray-worker-cert-delete-head-after-submit-r4xnr\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.498Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"601e44ae-9bb4-4935-98b8-4aca597139f0"}
{"level":"error","ts":"2026-09-18T10:03:21.498Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"601e44ae-9bb4-4935-98b8-4aca597139f0","error":"Certificate.cert-manager.io \"ray-worker-cert-delete-head-after-submit-r4xnr\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.498Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"82d1d5d4-6a38-480e-b1ea-07458cf40164","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.499Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"82d1d5d4-6a38-480e-b1ea-07458cf40164"}
{"level":"info","ts":"2026-09-18T10:03:21.504Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"13897c1f-d298-4842-94ef-8edcf2a5f93d","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.504Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"13897c1f-d298-4842-94ef-8edcf2a5f93d"}
{"level":"info","ts":"2026-09-18T10:03:21.506Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"ddd8a2a2-c167-49d8-84e5-74ac3e49aa0d","name":"delete-head-after-submit-r4xnr-head"}
{"level":"info","ts":"2026-09-18T10:03:21.509Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"73385ced-3b2c-4ff9-ac37-a9570e109812","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"ea5edf2a-ebc9-4cdd-843a-0640ed5222f8"}
{"level":"info","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"ea5edf2a-ebc9-4cdd-843a-0640ed5222f8"}
{"level":"info","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"ea5edf2a-ebc9-4cdd-843a-0640ed5222f8"}
{"level":"info","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"ea5edf2a-ebc9-4cdd-843a-0640ed5222f8","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"ea5edf2a-ebc9-4cdd-843a-0640ed5222f8","error":"head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"error","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"ea5edf2a-ebc9-4cdd-843a-0640ed5222f8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"ea5edf2a-ebc9-4cdd-843a-0640ed5222f8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"info","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"ea5edf2a-ebc9-4cdd-843a-0640ed5222f8","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"ea5edf2a-ebc9-4cdd-843a-0640ed5222f8","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"ea5edf2a-ebc9-4cdd-843a-0640ed5222f8","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"ea5edf2a-ebc9-4cdd-843a-0640ed5222f8"}
{"level":"error","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"ea5edf2a-ebc9-4cdd-843a-0640ed5222f8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"3aee1d47-f066-4e4d-af55-8796fc429b89","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"3aee1d47-f066-4e4d-af55-8796fc429b89","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"3aee1d47-f066-4e4d-af55-8796fc429b89","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"3aee1d47-f066-4e4d-af55-8796fc429b89"}
{"level":"info","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"3aee1d47-f066-4e4d-af55-8796fc429b89","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:21.566Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"3aee1d47-f066-4e4d-af55-8796fc429b89","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:21.573Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"ddd8a2a2-c167-49d8-84e5-74ac3e49aa0d","name":"delete-head-after-submit-r4xnr-workers"}
{"level":"info","ts":"2026-09-18T10:03:21.573Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"ddd8a2a2-c167-49d8-84e5-74ac3e49aa0d","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.573Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f2eb1ede-4988-46f2-a950-4e937b6c1644","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.589Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f2eb1ede-4988-46f2-a950-4e937b6c1644","name":"delete-head-after-submit-r4xnr-head"}
{"level":"info","ts":"2026-09-18T10:03:21.610Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f2eb1ede-4988-46f2-a950-4e937b6c1644","name":"delete-head-after-submit-r4xnr-workers"}
{"level":"info","ts":"2026-09-18T10:03:21.610Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f2eb1ede-4988-46f2-a950-4e937b6c1644","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:21.726Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"62fc8acd-96e3-4f97-b27d-883d70d1cf75"}
{"level":"info","ts":"2026-09-18T10:03:21.726Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"62fc8acd-96e3-4f97-b27d-883d70d1cf75"}
{"level":"info","ts":"2026-09-18T10:03:21.727Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"62fc8acd-96e3-4f97-b27d-883d70d1cf75"}
{"level":"info","ts":"2026-09-18T10:03:21.727Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"62fc8acd-96e3-4f97-b27d-883d70d1cf75","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:21.727Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"62fc8acd-96e3-4f97-b27d-883d70d1cf75","error":"head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"error","ts":"2026-09-18T10:03:21.727Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"62fc8acd-96e3-4f97-b27d-883d70d1cf75","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:21.727Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"62fc8acd-96e3-4f97-b27d-883d70d1cf75","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"info","ts":"2026-09-18T10:03:21.727Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"62fc8acd-96e3-4f97-b27d-883d70d1cf75","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.727Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"62fc8acd-96e3-4f97-b27d-883d70d1cf75","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.727Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"62fc8acd-96e3-4f97-b27d-883d70d1cf75","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:21.727Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"62fc8acd-96e3-4f97-b27d-883d70d1cf75"}
{"level":"error","ts":"2026-09-18T10:03:21.727Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"62fc8acd-96e3-4f97-b27d-883d70d1cf75","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:22.047Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7c7ced9c-0bc2-4104-88cd-c3eaa252fdc8"}
{"level":"info","ts":"2026-09-18T10:03:22.047Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7c7ced9c-0bc2-4104-88cd-c3eaa252fdc8"}
{"level":"info","ts":"2026-09-18T10:03:22.047Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7c7ced9c-0bc2-4104-88cd-c3eaa252fdc8"}
{"level":"info","ts":"2026-09-18T10:03:22.047Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7c7ced9c-0bc2-4104-88cd-c3eaa252fdc8","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:22.047Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7c7ced9c-0bc2-4104-88cd-c3eaa252fdc8","error":"head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"error","ts":"2026-09-18T10:03:22.047Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7c7ced9c-0bc2-4104-88cd-c3eaa252fdc8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:22.047Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7c7ced9c-0bc2-4104-88cd-c3eaa252fdc8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"info","ts":"2026-09-18T10:03:22.047Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7c7ced9c-0bc2-4104-88cd-c3eaa252fdc8","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:22.047Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7c7ced9c-0bc2-4104-88cd-c3eaa252fdc8","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:22.047Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7c7ced9c-0bc2-4104-88cd-c3eaa252fdc8","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:22.047Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7c7ced9c-0bc2-4104-88cd-c3eaa252fdc8"}
{"level":"error","ts":"2026-09-18T10:03:22.047Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7c7ced9c-0bc2-4104-88cd-c3eaa252fdc8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:22.660Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"14a5ed49-2127-41dc-bd63-43b13a6a9fa4"}
{"level":"info","ts":"2026-09-18T10:03:22.660Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"14a5ed49-2127-41dc-bd63-43b13a6a9fa4"}
{"level":"info","ts":"2026-09-18T10:03:22.660Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"14a5ed49-2127-41dc-bd63-43b13a6a9fa4"}
{"level":"info","ts":"2026-09-18T10:03:22.660Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"14a5ed49-2127-41dc-bd63-43b13a6a9fa4","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:22.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"14a5ed49-2127-41dc-bd63-43b13a6a9fa4","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:22.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"14a5ed49-2127-41dc-bd63-43b13a6a9fa4","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:22.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"14a5ed49-2127-41dc-bd63-43b13a6a9fa4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:22.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"14a5ed49-2127-41dc-bd63-43b13a6a9fa4","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:22.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"14a5ed49-2127-41dc-bd63-43b13a6a9fa4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:22.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"14a5ed49-2127-41dc-bd63-43b13a6a9fa4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:22.660Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"14a5ed49-2127-41dc-bd63-43b13a6a9fa4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:22.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"14a5ed49-2127-41dc-bd63-43b13a6a9fa4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:22.660Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"14a5ed49-2127-41dc-bd63-43b13a6a9fa4","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:22.660Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"14a5ed49-2127-41dc-bd63-43b13a6a9fa4","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:22.660Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"14a5ed49-2127-41dc-bd63-43b13a6a9fa4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:22.660Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"14a5ed49-2127-41dc-bd63-43b13a6a9fa4","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:22.688Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"330c5d8b-9afd-4b59-b52a-55975490b47c"}
{"level":"info","ts":"2026-09-18T10:03:22.688Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"330c5d8b-9afd-4b59-b52a-55975490b47c"}
{"level":"info","ts":"2026-09-18T10:03:22.688Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"330c5d8b-9afd-4b59-b52a-55975490b47c"}
{"level":"info","ts":"2026-09-18T10:03:22.688Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"330c5d8b-9afd-4b59-b52a-55975490b47c","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:22.688Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"330c5d8b-9afd-4b59-b52a-55975490b47c","error":"head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"error","ts":"2026-09-18T10:03:22.688Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"330c5d8b-9afd-4b59-b52a-55975490b47c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:22.688Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"330c5d8b-9afd-4b59-b52a-55975490b47c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"info","ts":"2026-09-18T10:03:22.689Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"330c5d8b-9afd-4b59-b52a-55975490b47c","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:22.689Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"330c5d8b-9afd-4b59-b52a-55975490b47c","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:22.689Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"330c5d8b-9afd-4b59-b52a-55975490b47c","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:22.689Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"330c5d8b-9afd-4b59-b52a-55975490b47c"}
{"level":"error","ts":"2026-09-18T10:03:22.689Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"330c5d8b-9afd-4b59-b52a-55975490b47c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:23.969Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b7dd56a-440d-4896-a3d6-72747744c79d"}
{"level":"info","ts":"2026-09-18T10:03:23.969Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b7dd56a-440d-4896-a3d6-72747744c79d"}
{"level":"info","ts":"2026-09-18T10:03:23.969Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b7dd56a-440d-4896-a3d6-72747744c79d"}
{"level":"info","ts":"2026-09-18T10:03:23.969Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b7dd56a-440d-4896-a3d6-72747744c79d","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:23.969Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b7dd56a-440d-4896-a3d6-72747744c79d","error":"head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"error","ts":"2026-09-18T10:03:23.969Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b7dd56a-440d-4896-a3d6-72747744c79d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:23.969Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b7dd56a-440d-4896-a3d6-72747744c79d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found"}
{"level":"info","ts":"2026-09-18T10:03:23.969Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b7dd56a-440d-4896-a3d6-72747744c79d","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:23.969Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b7dd56a-440d-4896-a3d6-72747744c79d","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:23.969Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b7dd56a-440d-4896-a3d6-72747744c79d","filter labels":["test-ns-ltx56",{"ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:03:23.969Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b7dd56a-440d-4896-a3d6-72747744c79d"}
{"level":"error","ts":"2026-09-18T10:03:23.969Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b7dd56a-440d-4896-a3d6-72747744c79d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-delete-head-after-submit-r4xnr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:24.305Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2bd7081b-ff1e-4b98-a1e5-1fde2696cb66","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:24.305Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2bd7081b-ff1e-4b98-a1e5-1fde2696cb66","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:24.305Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2bd7081b-ff1e-4b98-a1e5-1fde2696cb66","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:24.305Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2bd7081b-ff1e-4b98-a1e5-1fde2696cb66"}
{"level":"info","ts":"2026-09-18T10:03:24.305Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2bd7081b-ff1e-4b98-a1e5-1fde2696cb66","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:24.305Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2bd7081b-ff1e-4b98-a1e5-1fde2696cb66","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:25.668Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"9be538d2-46d5-4596-a700-50620d6142fd"}
{"level":"info","ts":"2026-09-18T10:03:25.668Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"9be538d2-46d5-4596-a700-50620d6142fd"}
{"level":"info","ts":"2026-09-18T10:03:25.668Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"9be538d2-46d5-4596-a700-50620d6142fd"}
{"level":"info","ts":"2026-09-18T10:03:25.669Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"9be538d2-46d5-4596-a700-50620d6142fd","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:25.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"9be538d2-46d5-4596-a700-50620d6142fd","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:25.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"9be538d2-46d5-4596-a700-50620d6142fd","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:25.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"9be538d2-46d5-4596-a700-50620d6142fd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:25.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"9be538d2-46d5-4596-a700-50620d6142fd","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:25.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"9be538d2-46d5-4596-a700-50620d6142fd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:25.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"9be538d2-46d5-4596-a700-50620d6142fd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:25.669Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"9be538d2-46d5-4596-a700-50620d6142fd","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:25.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"9be538d2-46d5-4596-a700-50620d6142fd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:25.669Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"9be538d2-46d5-4596-a700-50620d6142fd","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:25.669Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"9be538d2-46d5-4596-a700-50620d6142fd","podName":"long-running-5qfl8-head-kflfc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:25.669Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"9be538d2-46d5-4596-a700-50620d6142fd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:25.669Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"9be538d2-46d5-4596-a700-50620d6142fd","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:26.531Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee"}
{"level":"info","ts":"2026-09-18T10:03:26.531Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee"}
{"level":"info","ts":"2026-09-18T10:03:26.531Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee"}
{"level":"info","ts":"2026-09-18T10:03:26.531Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:26.531Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee"}
{"level":"info","ts":"2026-09-18T10:03:26.531Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:03:26.531Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee"}
{"level":"info","ts":"2026-09-18T10:03:26.531Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","mode":"IntegratedOAuth","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:26.531Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","cluster":"delete-head-after-submit-r4xnr","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:03:26.531Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","cluster":"delete-head-after-submit-r4xnr","authType":"IntegratedOAuth","serviceAccount":"delete-head-after-submit-r4xnr-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T10:03:26.531Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/group":"headgroup","ray.io/identifier":"delete-head-after-submit-r4xnr-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:03:26.531Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T10:03:26.531Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:03:26.531Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:03:26.531Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","enabled":true}
{"level":"info","ts":"2026-09-18T10:03:26.556Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","name":"delete-head-after-submit-r4xnr-head-kw54w"}
{"level":"info","ts":"2026-09-18T10:03:26.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:26.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:26.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:03:26.556Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:03:26.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:26.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:03:26.556Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee"}
{"level":"info","ts":"2026-09-18T10:03:26.556Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","nodeType":"worker","rayStartParams":{"address":"delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:03:26.556Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","rayStartCmd":"ray start  --address=delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:03:26.556Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:03:26.556Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","enabled":true}
{"level":"info","ts":"2026-09-18T10:03:26.591Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","name":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm"}
{"level":"info","ts":"2026-09-18T10:03:26.592Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:26.592Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:26.592Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b298aab-606e-4aa9-9521-8297ad7a00ee","name":"delete-head-after-submit-r4xnr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:03:26.617Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"69afe7d3-263c-45ba-9a6f-11e39f69e858","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:26.617Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"69afe7d3-263c-45ba-9a6f-11e39f69e858","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:26.617Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"86d985b7-cc87-4c69-aeb6-90d1a3a785a4","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:26.617Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"69afe7d3-263c-45ba-9a6f-11e39f69e858","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:26.618Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"86d985b7-cc87-4c69-aeb6-90d1a3a785a4","rayCluster":"delete-head-after-submit-r4xnr","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:03:26.618Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"69afe7d3-263c-45ba-9a6f-11e39f69e858"}
{"level":"info","ts":"2026-09-18T10:03:26.618Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"aa9d190f-37b6-42bb-b762-1ba5e8081173","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:26.618Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"69afe7d3-263c-45ba-9a6f-11e39f69e858","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:26.618Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"69afe7d3-263c-45ba-9a6f-11e39f69e858","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:26.618Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"69afe7d3-263c-45ba-9a6f-11e39f69e858","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:03:26.619Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4f90a719-3b1d-4e42-8c34-97ea1d4abb37"}
{"level":"info","ts":"2026-09-18T10:03:26.619Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4f90a719-3b1d-4e42-8c34-97ea1d4abb37"}
{"level":"info","ts":"2026-09-18T10:03:26.619Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4f90a719-3b1d-4e42-8c34-97ea1d4abb37"}
{"level":"info","ts":"2026-09-18T10:03:26.619Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4f90a719-3b1d-4e42-8c34-97ea1d4abb37","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:26.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4f90a719-3b1d-4e42-8c34-97ea1d4abb37","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:26.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4f90a719-3b1d-4e42-8c34-97ea1d4abb37","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:26.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4f90a719-3b1d-4e42-8c34-97ea1d4abb37","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:26.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4f90a719-3b1d-4e42-8c34-97ea1d4abb37","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:26.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4f90a719-3b1d-4e42-8c34-97ea1d4abb37","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:26.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4f90a719-3b1d-4e42-8c34-97ea1d4abb37","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:26.619Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4f90a719-3b1d-4e42-8c34-97ea1d4abb37","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:26.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4f90a719-3b1d-4e42-8c34-97ea1d4abb37","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:26.619Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4f90a719-3b1d-4e42-8c34-97ea1d4abb37","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:26.619Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4f90a719-3b1d-4e42-8c34-97ea1d4abb37","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:26.619Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4f90a719-3b1d-4e42-8c34-97ea1d4abb37","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:26.619Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4f90a719-3b1d-4e42-8c34-97ea1d4abb37","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:26.640Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"aa9d190f-37b6-42bb-b762-1ba5e8081173","name":"delete-head-after-submit-r4xnr-head"}
{"level":"info","ts":"2026-09-18T10:03:26.642Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9bcbf4f5-e4e1-428c-8334-2e6a506277c4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:26.642Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9bcbf4f5-e4e1-428c-8334-2e6a506277c4","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:26.642Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9bcbf4f5-e4e1-428c-8334-2e6a506277c4","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:26.642Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9bcbf4f5-e4e1-428c-8334-2e6a506277c4"}
{"level":"info","ts":"2026-09-18T10:03:26.642Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9bcbf4f5-e4e1-428c-8334-2e6a506277c4","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:26.642Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9bcbf4f5-e4e1-428c-8334-2e6a506277c4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:26.645Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e1e4f59d-b1e8-464d-a308-e2a1b0d278a8"}
{"level":"info","ts":"2026-09-18T10:03:26.645Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e1e4f59d-b1e8-464d-a308-e2a1b0d278a8"}
{"level":"info","ts":"2026-09-18T10:03:26.645Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e1e4f59d-b1e8-464d-a308-e2a1b0d278a8"}
{"level":"info","ts":"2026-09-18T10:03:26.645Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e1e4f59d-b1e8-464d-a308-e2a1b0d278a8","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:26.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e1e4f59d-b1e8-464d-a308-e2a1b0d278a8","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:26.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e1e4f59d-b1e8-464d-a308-e2a1b0d278a8","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:26.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e1e4f59d-b1e8-464d-a308-e2a1b0d278a8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:26.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e1e4f59d-b1e8-464d-a308-e2a1b0d278a8","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:26.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e1e4f59d-b1e8-464d-a308-e2a1b0d278a8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:26.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e1e4f59d-b1e8-464d-a308-e2a1b0d278a8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:26.645Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e1e4f59d-b1e8-464d-a308-e2a1b0d278a8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:26.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e1e4f59d-b1e8-464d-a308-e2a1b0d278a8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:26.645Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e1e4f59d-b1e8-464d-a308-e2a1b0d278a8","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:26.645Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e1e4f59d-b1e8-464d-a308-e2a1b0d278a8","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:26.645Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e1e4f59d-b1e8-464d-a308-e2a1b0d278a8","name":"delete-head-after-submit-r4xnr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:03:26.682Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"aa9d190f-37b6-42bb-b762-1ba5e8081173","name":"delete-head-after-submit-r4xnr-workers"}
{"level":"info","ts":"2026-09-18T10:03:26.682Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"aa9d190f-37b6-42bb-b762-1ba5e8081173","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:26.684Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d"}
{"level":"info","ts":"2026-09-18T10:03:26.684Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d"}
{"level":"info","ts":"2026-09-18T10:03:26.684Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d"}
{"level":"info","ts":"2026-09-18T10:03:26.684Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"779dbfca-b052-4c03-9051-56447822ef13","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"779dbfca-b052-4c03-9051-56447822ef13","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"779dbfca-b052-4c03-9051-56447822ef13","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"e5bb7fcf-bc1b-49d6-8ff2-eddb90157117","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"779dbfca-b052-4c03-9051-56447822ef13"}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"779dbfca-b052-4c03-9051-56447822ef13","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"779dbfca-b052-4c03-9051-56447822ef13","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"779dbfca-b052-4c03-9051-56447822ef13","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"351c04cc-2b0e-4c20-9007-3a6662b8f47d","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"351c04cc-2b0e-4c20-9007-3a6662b8f47d","rayCluster":"delete-head-after-submit-r4xnr","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:26.685Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d","name":"delete-head-after-submit-r4xnr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:03:26.702Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d","name":"delete-head-after-submit-r4xnr","error":"Operation cannot be fulfilled on rayclusters.ray.io \"delete-head-after-submit-r4xnr\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr"}}
{"level":"info","ts":"2026-09-18T10:03:26.702Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d"}
{"level":"error","ts":"2026-09-18T10:03:26.702Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f141ac73-7f28-4c84-a4ed-29a766d56c0d","error":"Operation cannot be fulfilled on rayclusters.ray.io \"delete-head-after-submit-r4xnr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:03:26.708Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"94817594-2074-4ef4-bdc4-deeba6b51a70"}
{"level":"info","ts":"2026-09-18T10:03:26.708Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"94817594-2074-4ef4-bdc4-deeba6b51a70"}
{"level":"info","ts":"2026-09-18T10:03:26.708Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"94817594-2074-4ef4-bdc4-deeba6b51a70"}
{"level":"info","ts":"2026-09-18T10:03:26.708Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"94817594-2074-4ef4-bdc4-deeba6b51a70","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:26.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"94817594-2074-4ef4-bdc4-deeba6b51a70","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:26.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"94817594-2074-4ef4-bdc4-deeba6b51a70","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:26.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"94817594-2074-4ef4-bdc4-deeba6b51a70","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:26.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"94817594-2074-4ef4-bdc4-deeba6b51a70","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:26.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"94817594-2074-4ef4-bdc4-deeba6b51a70","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:26.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"94817594-2074-4ef4-bdc4-deeba6b51a70","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:26.709Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"94817594-2074-4ef4-bdc4-deeba6b51a70","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:26.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"94817594-2074-4ef4-bdc4-deeba6b51a70","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:26.709Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"94817594-2074-4ef4-bdc4-deeba6b51a70","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:26.709Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"94817594-2074-4ef4-bdc4-deeba6b51a70","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:26.709Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"94817594-2074-4ef4-bdc4-deeba6b51a70","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:26.709Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"94817594-2074-4ef4-bdc4-deeba6b51a70","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:26.719Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"e5bb7fcf-bc1b-49d6-8ff2-eddb90157117","name":"delete-head-after-submit-r4xnr-head"}
{"level":"info","ts":"2026-09-18T10:03:26.720Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"21d2b2d6-cf70-4e35-86ac-52e20efda9b7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:26.720Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"21d2b2d6-cf70-4e35-86ac-52e20efda9b7","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:26.720Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"21d2b2d6-cf70-4e35-86ac-52e20efda9b7","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:26.720Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"21d2b2d6-cf70-4e35-86ac-52e20efda9b7"}
{"level":"info","ts":"2026-09-18T10:03:26.720Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"21d2b2d6-cf70-4e35-86ac-52e20efda9b7","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:26.720Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"21d2b2d6-cf70-4e35-86ac-52e20efda9b7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:26.731Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8cfe8d06-f82c-44b3-8f81-64a6fe53d65e"}
{"level":"info","ts":"2026-09-18T10:03:26.731Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8cfe8d06-f82c-44b3-8f81-64a6fe53d65e"}
{"level":"info","ts":"2026-09-18T10:03:26.731Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8cfe8d06-f82c-44b3-8f81-64a6fe53d65e"}
{"level":"info","ts":"2026-09-18T10:03:26.731Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8cfe8d06-f82c-44b3-8f81-64a6fe53d65e","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:26.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8cfe8d06-f82c-44b3-8f81-64a6fe53d65e","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:26.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8cfe8d06-f82c-44b3-8f81-64a6fe53d65e","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:26.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8cfe8d06-f82c-44b3-8f81-64a6fe53d65e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:26.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8cfe8d06-f82c-44b3-8f81-64a6fe53d65e","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:26.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8cfe8d06-f82c-44b3-8f81-64a6fe53d65e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:26.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8cfe8d06-f82c-44b3-8f81-64a6fe53d65e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:26.731Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8cfe8d06-f82c-44b3-8f81-64a6fe53d65e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:26.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8cfe8d06-f82c-44b3-8f81-64a6fe53d65e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:26.731Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8cfe8d06-f82c-44b3-8f81-64a6fe53d65e","podName":"long-running-5qfl8-head-kflfc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:03:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:03:26.731Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8cfe8d06-f82c-44b3-8f81-64a6fe53d65e","podName":"long-running-5qfl8-head-kflfc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:03:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:03:26.731Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8cfe8d06-f82c-44b3-8f81-64a6fe53d65e","name":"long-running-5qfl8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.42","serviceIP":"10.133.0.42","podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.42","serviceIP":"10.133.0.42","podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:03:26.766Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"e5bb7fcf-bc1b-49d6-8ff2-eddb90157117","name":"delete-head-after-submit-r4xnr-workers"}
{"level":"info","ts":"2026-09-18T10:03:26.766Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"e5bb7fcf-bc1b-49d6-8ff2-eddb90157117","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:26.766Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"1f59e23c-e9ce-4930-b2e7-f2e7005ca869","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:26.766Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"b1130896-21be-44b9-b0b3-ec7efb48c828","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:26.767Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"b1130896-21be-44b9-b0b3-ec7efb48c828","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:03:26.767Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"b1130896-21be-44b9-b0b3-ec7efb48c828","pod":"long-running-5qfl8-small-group-worker-tbqtj","ip":"10.132.0.47"}
{"level":"info","ts":"2026-09-18T10:03:26.767Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"b1130896-21be-44b9-b0b3-ec7efb48c828","rayCluster":"long-running-5qfl8","podIPs":["10.133.0.42","10.132.0.47"]}
{"level":"info","ts":"2026-09-18T10:03:26.767Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"30bfb2cb-5df8-4629-bd81-6441b01a00e1","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:26.767Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"30bfb2cb-5df8-4629-bd81-6441b01a00e1","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-09-18 10:03:20 +0000 UTC","Now":"2026-09-18T10:03:26.767Z","ShutdownTime":"2026-09-18T10:13:20.000Z"}
{"level":"info","ts":"2026-09-18T10:03:26.767Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"30bfb2cb-5df8-4629-bd81-6441b01a00e1","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-09-18T10:13:20.000Z","requeueAfter":595.232754445}
{"level":"info","ts":"2026-09-18T10:03:26.785Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"1f59e23c-e9ce-4930-b2e7-f2e7005ca869","name":"long-running-5qfl8-head"}
{"level":"info","ts":"2026-09-18T10:03:26.800Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"1f59e23c-e9ce-4930-b2e7-f2e7005ca869","name":"long-running-5qfl8-workers"}
{"level":"info","ts":"2026-09-18T10:03:26.800Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"1f59e23c-e9ce-4930-b2e7-f2e7005ca869","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:27.162Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a497d1a-f538-4c4d-9cec-da59247e3ea0"}
{"level":"info","ts":"2026-09-18T10:03:27.162Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a497d1a-f538-4c4d-9cec-da59247e3ea0"}
{"level":"info","ts":"2026-09-18T10:03:27.162Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a497d1a-f538-4c4d-9cec-da59247e3ea0"}
{"level":"info","ts":"2026-09-18T10:03:27.163Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a497d1a-f538-4c4d-9cec-da59247e3ea0","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:27.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a497d1a-f538-4c4d-9cec-da59247e3ea0","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:27.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a497d1a-f538-4c4d-9cec-da59247e3ea0","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:27.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a497d1a-f538-4c4d-9cec-da59247e3ea0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:27.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a497d1a-f538-4c4d-9cec-da59247e3ea0","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:27.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a497d1a-f538-4c4d-9cec-da59247e3ea0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:27.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a497d1a-f538-4c4d-9cec-da59247e3ea0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:27.163Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a497d1a-f538-4c4d-9cec-da59247e3ea0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:27.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a497d1a-f538-4c4d-9cec-da59247e3ea0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:27.163Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a497d1a-f538-4c4d-9cec-da59247e3ea0","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:27.163Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a497d1a-f538-4c4d-9cec-da59247e3ea0","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:27.163Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a497d1a-f538-4c4d-9cec-da59247e3ea0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:27.163Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a497d1a-f538-4c4d-9cec-da59247e3ea0","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:27.217Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"3001fb4f-d6b7-4295-b67e-96c8b5046f99"}
{"level":"info","ts":"2026-09-18T10:03:27.217Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"3001fb4f-d6b7-4295-b67e-96c8b5046f99"}
{"level":"info","ts":"2026-09-18T10:03:27.217Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"3001fb4f-d6b7-4295-b67e-96c8b5046f99"}
{"level":"info","ts":"2026-09-18T10:03:27.217Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"3001fb4f-d6b7-4295-b67e-96c8b5046f99","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:27.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"3001fb4f-d6b7-4295-b67e-96c8b5046f99","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:27.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"3001fb4f-d6b7-4295-b67e-96c8b5046f99","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:27.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"3001fb4f-d6b7-4295-b67e-96c8b5046f99","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:27.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"3001fb4f-d6b7-4295-b67e-96c8b5046f99","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:27.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"3001fb4f-d6b7-4295-b67e-96c8b5046f99","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:27.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"3001fb4f-d6b7-4295-b67e-96c8b5046f99","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:27.218Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"3001fb4f-d6b7-4295-b67e-96c8b5046f99","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:27.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"3001fb4f-d6b7-4295-b67e-96c8b5046f99","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:27.218Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"3001fb4f-d6b7-4295-b67e-96c8b5046f99","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:27.218Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"3001fb4f-d6b7-4295-b67e-96c8b5046f99","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:27.218Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"3001fb4f-d6b7-4295-b67e-96c8b5046f99","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:27.218Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"3001fb4f-d6b7-4295-b67e-96c8b5046f99","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:27.271Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"92e1bfc3-f29c-495e-820e-be4cd8b6561f"}
{"level":"info","ts":"2026-09-18T10:03:27.271Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"92e1bfc3-f29c-495e-820e-be4cd8b6561f"}
{"level":"info","ts":"2026-09-18T10:03:27.271Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"92e1bfc3-f29c-495e-820e-be4cd8b6561f"}
{"level":"info","ts":"2026-09-18T10:03:27.271Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"92e1bfc3-f29c-495e-820e-be4cd8b6561f","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:27.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"92e1bfc3-f29c-495e-820e-be4cd8b6561f","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:27.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"92e1bfc3-f29c-495e-820e-be4cd8b6561f","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:27.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"92e1bfc3-f29c-495e-820e-be4cd8b6561f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:27.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"92e1bfc3-f29c-495e-820e-be4cd8b6561f","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:27.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"92e1bfc3-f29c-495e-820e-be4cd8b6561f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:27.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"92e1bfc3-f29c-495e-820e-be4cd8b6561f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:27.271Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"92e1bfc3-f29c-495e-820e-be4cd8b6561f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:27.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"92e1bfc3-f29c-495e-820e-be4cd8b6561f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:27.271Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"92e1bfc3-f29c-495e-820e-be4cd8b6561f","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:27.272Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"92e1bfc3-f29c-495e-820e-be4cd8b6561f","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:27.272Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"92e1bfc3-f29c-495e-820e-be4cd8b6561f","name":"delete-head-after-submit-r4xnr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:03:27.290Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"7213c12c-fe9a-46a1-86ae-88829b23f465","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:27.290Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ea338434-a34a-4697-b305-06c26edbdc6d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:27.290Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ea338434-a34a-4697-b305-06c26edbdc6d","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:27.290Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ea338434-a34a-4697-b305-06c26edbdc6d","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:27.290Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ea338434-a34a-4697-b305-06c26edbdc6d"}
{"level":"info","ts":"2026-09-18T10:03:27.290Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ea338434-a34a-4697-b305-06c26edbdc6d","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:27.290Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ea338434-a34a-4697-b305-06c26edbdc6d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:27.290Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ea338434-a34a-4697-b305-06c26edbdc6d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:03:27.291Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"26cd208f-9350-439b-b2b3-fd093780ce87","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:27.291Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"26cd208f-9350-439b-b2b3-fd093780ce87","pod":"delete-head-after-submit-r4xnr-head-kw54w","ip":"10.132.0.48"}
{"level":"info","ts":"2026-09-18T10:03:27.309Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"085dd1be-3bc6-4510-b6d2-aaf13b27b05e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:27.309Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"26cd208f-9350-439b-b2b3-fd093780ce87","certificate":"ray-head-cert-delete-head-after-submit-r4xnr","dnsNames":["delete-head-after-submit-r4xnr-head-svc","localhost","delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc","delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.48","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:03:27.309Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"085dd1be-3bc6-4510-b6d2-aaf13b27b05e","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:27.309Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"085dd1be-3bc6-4510-b6d2-aaf13b27b05e","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:27.310Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"085dd1be-3bc6-4510-b6d2-aaf13b27b05e"}
{"level":"info","ts":"2026-09-18T10:03:27.310Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"085dd1be-3bc6-4510-b6d2-aaf13b27b05e","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:27.310Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"085dd1be-3bc6-4510-b6d2-aaf13b27b05e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:27.311Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"7213c12c-fe9a-46a1-86ae-88829b23f465","name":"delete-head-after-submit-r4xnr-head"}
{"level":"info","ts":"2026-09-18T10:03:27.326Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"7213c12c-fe9a-46a1-86ae-88829b23f465","name":"delete-head-after-submit-r4xnr-workers"}
{"level":"info","ts":"2026-09-18T10:03:27.326Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"7213c12c-fe9a-46a1-86ae-88829b23f465","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:27.330Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"26cd208f-9350-439b-b2b3-fd093780ce87","certificate":"ray-worker-cert-delete-head-after-submit-r4xnr","dnsNames":["delete-head-after-submit-r4xnr-worker-svc","localhost","delete-head-after-submit-r4xnr-small-group","delete-head-after-submit-r4xnr-worker-svc.test-ns-ltx56.svc","delete-head-after-submit-r4xnr-worker-svc.test-ns-ltx56.svc.cluster.local","*.delete-head-after-submit-r4xnr-worker-svc.test-ns-ltx56.svc","*.delete-head-after-submit-r4xnr-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.48","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:03:27.330Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"26cd208f-9350-439b-b2b3-fd093780ce87"}
{"level":"info","ts":"2026-09-18T10:03:27.679Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"254729e8-2de4-4a9c-9d40-a897bb02d809"}
{"level":"info","ts":"2026-09-18T10:03:27.679Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"254729e8-2de4-4a9c-9d40-a897bb02d809"}
{"level":"info","ts":"2026-09-18T10:03:27.679Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"254729e8-2de4-4a9c-9d40-a897bb02d809"}
{"level":"info","ts":"2026-09-18T10:03:27.679Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"254729e8-2de4-4a9c-9d40-a897bb02d809","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:27.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"254729e8-2de4-4a9c-9d40-a897bb02d809","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:27.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"254729e8-2de4-4a9c-9d40-a897bb02d809","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:27.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"254729e8-2de4-4a9c-9d40-a897bb02d809","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:27.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"254729e8-2de4-4a9c-9d40-a897bb02d809","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:27.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"254729e8-2de4-4a9c-9d40-a897bb02d809","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:27.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"254729e8-2de4-4a9c-9d40-a897bb02d809","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:27.679Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"254729e8-2de4-4a9c-9d40-a897bb02d809","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:27.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"254729e8-2de4-4a9c-9d40-a897bb02d809","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:27.680Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"254729e8-2de4-4a9c-9d40-a897bb02d809","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:27.680Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"254729e8-2de4-4a9c-9d40-a897bb02d809","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:27.680Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"254729e8-2de4-4a9c-9d40-a897bb02d809","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:27.680Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"254729e8-2de4-4a9c-9d40-a897bb02d809","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:28.767Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"cd09560d-7009-4906-b482-e6c5bb47139c"}
{"level":"info","ts":"2026-09-18T10:03:28.767Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"cd09560d-7009-4906-b482-e6c5bb47139c"}
{"level":"info","ts":"2026-09-18T10:03:28.767Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"cd09560d-7009-4906-b482-e6c5bb47139c"}
{"level":"info","ts":"2026-09-18T10:03:28.767Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"cd09560d-7009-4906-b482-e6c5bb47139c","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:28.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"cd09560d-7009-4906-b482-e6c5bb47139c","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:28.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"cd09560d-7009-4906-b482-e6c5bb47139c","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:28.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"cd09560d-7009-4906-b482-e6c5bb47139c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:28.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"cd09560d-7009-4906-b482-e6c5bb47139c","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:28.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"cd09560d-7009-4906-b482-e6c5bb47139c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:28.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"cd09560d-7009-4906-b482-e6c5bb47139c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:28.767Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"cd09560d-7009-4906-b482-e6c5bb47139c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:28.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"cd09560d-7009-4906-b482-e6c5bb47139c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:28.768Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"cd09560d-7009-4906-b482-e6c5bb47139c","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:28.768Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"cd09560d-7009-4906-b482-e6c5bb47139c","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:28.768Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"cd09560d-7009-4906-b482-e6c5bb47139c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:28.768Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"cd09560d-7009-4906-b482-e6c5bb47139c","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:29.290Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"36c60230-1bcf-442f-8adf-b2de8c8ee010"}
{"level":"info","ts":"2026-09-18T10:03:29.290Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"36c60230-1bcf-442f-8adf-b2de8c8ee010"}
{"level":"info","ts":"2026-09-18T10:03:29.290Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"36c60230-1bcf-442f-8adf-b2de8c8ee010"}
{"level":"info","ts":"2026-09-18T10:03:29.290Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"36c60230-1bcf-442f-8adf-b2de8c8ee010","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:29.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"36c60230-1bcf-442f-8adf-b2de8c8ee010","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:29.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"36c60230-1bcf-442f-8adf-b2de8c8ee010","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:29.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"36c60230-1bcf-442f-8adf-b2de8c8ee010","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:29.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"36c60230-1bcf-442f-8adf-b2de8c8ee010","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:29.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"36c60230-1bcf-442f-8adf-b2de8c8ee010","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:29.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"36c60230-1bcf-442f-8adf-b2de8c8ee010","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:29.290Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"36c60230-1bcf-442f-8adf-b2de8c8ee010","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:29.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"36c60230-1bcf-442f-8adf-b2de8c8ee010","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:29.290Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"36c60230-1bcf-442f-8adf-b2de8c8ee010","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:29.290Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"36c60230-1bcf-442f-8adf-b2de8c8ee010","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:29.290Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"36c60230-1bcf-442f-8adf-b2de8c8ee010","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:29.290Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"36c60230-1bcf-442f-8adf-b2de8c8ee010","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:30.310Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"1f1b9c57-ee02-4724-bb68-098db3bb92b8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:30.310Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"1f1b9c57-ee02-4724-bb68-098db3bb92b8","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:30.310Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"1f1b9c57-ee02-4724-bb68-098db3bb92b8","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:30.310Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"1f1b9c57-ee02-4724-bb68-098db3bb92b8"}
{"level":"info","ts":"2026-09-18T10:03:30.310Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"1f1b9c57-ee02-4724-bb68-098db3bb92b8","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:30.310Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"1f1b9c57-ee02-4724-bb68-098db3bb92b8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:33.311Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"c5f0df8a-9c68-4524-aa5b-a0a8dd012e5b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:33.311Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"c5f0df8a-9c68-4524-aa5b-a0a8dd012e5b","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:33.311Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"c5f0df8a-9c68-4524-aa5b-a0a8dd012e5b","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:33.311Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"c5f0df8a-9c68-4524-aa5b-a0a8dd012e5b"}
{"level":"info","ts":"2026-09-18T10:03:33.311Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"c5f0df8a-9c68-4524-aa5b-a0a8dd012e5b","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:33.311Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"c5f0df8a-9c68-4524-aa5b-a0a8dd012e5b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:33.312Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"cd6fabac-7ea6-4c94-bbb6-e496fc02fb0b"}
{"level":"info","ts":"2026-09-18T10:03:33.312Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"cd6fabac-7ea6-4c94-bbb6-e496fc02fb0b"}
{"level":"info","ts":"2026-09-18T10:03:33.312Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"cd6fabac-7ea6-4c94-bbb6-e496fc02fb0b"}
{"level":"info","ts":"2026-09-18T10:03:33.312Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"cd6fabac-7ea6-4c94-bbb6-e496fc02fb0b","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:33.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"cd6fabac-7ea6-4c94-bbb6-e496fc02fb0b","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:33.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"cd6fabac-7ea6-4c94-bbb6-e496fc02fb0b","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:33.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"cd6fabac-7ea6-4c94-bbb6-e496fc02fb0b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:33.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"cd6fabac-7ea6-4c94-bbb6-e496fc02fb0b","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:33.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"cd6fabac-7ea6-4c94-bbb6-e496fc02fb0b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:33.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"cd6fabac-7ea6-4c94-bbb6-e496fc02fb0b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:33.312Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"cd6fabac-7ea6-4c94-bbb6-e496fc02fb0b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:33.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"cd6fabac-7ea6-4c94-bbb6-e496fc02fb0b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:33.312Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"cd6fabac-7ea6-4c94-bbb6-e496fc02fb0b","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:33.312Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"cd6fabac-7ea6-4c94-bbb6-e496fc02fb0b","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:33.312Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"cd6fabac-7ea6-4c94-bbb6-e496fc02fb0b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:33.312Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"cd6fabac-7ea6-4c94-bbb6-e496fc02fb0b","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:34.322Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"255ce683-d499-47ab-a71b-d9dd0283457d"}
{"level":"info","ts":"2026-09-18T10:03:34.322Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"255ce683-d499-47ab-a71b-d9dd0283457d"}
{"level":"info","ts":"2026-09-18T10:03:34.322Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"255ce683-d499-47ab-a71b-d9dd0283457d"}
{"level":"info","ts":"2026-09-18T10:03:34.322Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"255ce683-d499-47ab-a71b-d9dd0283457d","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:34.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"255ce683-d499-47ab-a71b-d9dd0283457d","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:34.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"255ce683-d499-47ab-a71b-d9dd0283457d","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:34.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"255ce683-d499-47ab-a71b-d9dd0283457d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:34.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"255ce683-d499-47ab-a71b-d9dd0283457d","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:34.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"255ce683-d499-47ab-a71b-d9dd0283457d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:34.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"255ce683-d499-47ab-a71b-d9dd0283457d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:34.322Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"255ce683-d499-47ab-a71b-d9dd0283457d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:34.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"255ce683-d499-47ab-a71b-d9dd0283457d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:34.322Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"255ce683-d499-47ab-a71b-d9dd0283457d","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:03:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:03:34.322Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"255ce683-d499-47ab-a71b-d9dd0283457d","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:03:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:03:34.322Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"255ce683-d499-47ab-a71b-d9dd0283457d","name":"delete-head-after-submit-r4xnr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:03:34.337Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"0ba1f8f4-f051-4750-8413-9a7b83e57037","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:34.337Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"0e09bd84-6427-41fa-99c9-f5a5c6691190","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:34.337Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"0e09bd84-6427-41fa-99c9-f5a5c6691190","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:34.337Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"0e09bd84-6427-41fa-99c9-f5a5c6691190","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:34.337Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"da846768-f7a3-431a-9567-42fc01f7b104","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:34.337Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"0e09bd84-6427-41fa-99c9-f5a5c6691190"}
{"level":"info","ts":"2026-09-18T10:03:34.337Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"0e09bd84-6427-41fa-99c9-f5a5c6691190","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:34.337Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"0e09bd84-6427-41fa-99c9-f5a5c6691190","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:34.337Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"0e09bd84-6427-41fa-99c9-f5a5c6691190","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:03:34.337Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"da846768-f7a3-431a-9567-42fc01f7b104","pod":"delete-head-after-submit-r4xnr-head-kw54w","ip":"10.132.0.48"}
{"level":"info","ts":"2026-09-18T10:03:34.337Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"da846768-f7a3-431a-9567-42fc01f7b104","pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","ip":"10.133.0.43"}
{"level":"info","ts":"2026-09-18T10:03:34.355Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef52c30d-d3e0-40c2-9c0c-57dc9ece3352","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:34.355Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef52c30d-d3e0-40c2-9c0c-57dc9ece3352","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:34.355Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef52c30d-d3e0-40c2-9c0c-57dc9ece3352","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:34.355Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef52c30d-d3e0-40c2-9c0c-57dc9ece3352"}
{"level":"info","ts":"2026-09-18T10:03:34.355Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef52c30d-d3e0-40c2-9c0c-57dc9ece3352","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:34.355Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef52c30d-d3e0-40c2-9c0c-57dc9ece3352","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:34.359Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"da846768-f7a3-431a-9567-42fc01f7b104","certificate":"ray-head-cert-delete-head-after-submit-r4xnr","dnsNames":["delete-head-after-submit-r4xnr-head-svc","localhost","delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc","delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.48","10.133.0.43","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:03:34.365Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"0ba1f8f4-f051-4750-8413-9a7b83e57037","name":"delete-head-after-submit-r4xnr-head"}
{"level":"info","ts":"2026-09-18T10:03:34.380Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"da846768-f7a3-431a-9567-42fc01f7b104","certificate":"ray-worker-cert-delete-head-after-submit-r4xnr","dnsNames":["delete-head-after-submit-r4xnr-worker-svc","localhost","delete-head-after-submit-r4xnr-small-group","delete-head-after-submit-r4xnr-worker-svc.test-ns-ltx56.svc","delete-head-after-submit-r4xnr-worker-svc.test-ns-ltx56.svc.cluster.local","*.delete-head-after-submit-r4xnr-worker-svc.test-ns-ltx56.svc","*.delete-head-after-submit-r4xnr-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.48","10.133.0.43","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:03:34.380Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"da846768-f7a3-431a-9567-42fc01f7b104"}
{"level":"info","ts":"2026-09-18T10:03:34.387Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"0ba1f8f4-f051-4750-8413-9a7b83e57037","name":"delete-head-after-submit-r4xnr-workers"}
{"level":"info","ts":"2026-09-18T10:03:34.387Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"0ba1f8f4-f051-4750-8413-9a7b83e57037","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:36.312Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"180f126a-7a2d-4f69-afde-20b482c3a7f3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:36.312Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"180f126a-7a2d-4f69-afde-20b482c3a7f3","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:36.312Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"180f126a-7a2d-4f69-afde-20b482c3a7f3","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:36.312Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"180f126a-7a2d-4f69-afde-20b482c3a7f3"}
{"level":"info","ts":"2026-09-18T10:03:36.312Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"180f126a-7a2d-4f69-afde-20b482c3a7f3","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:36.312Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"180f126a-7a2d-4f69-afde-20b482c3a7f3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:36.337Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"023320e4-5162-4756-a05b-2d501649b77a"}
{"level":"info","ts":"2026-09-18T10:03:36.337Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"023320e4-5162-4756-a05b-2d501649b77a"}
{"level":"info","ts":"2026-09-18T10:03:36.337Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"023320e4-5162-4756-a05b-2d501649b77a"}
{"level":"info","ts":"2026-09-18T10:03:36.337Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"023320e4-5162-4756-a05b-2d501649b77a","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:36.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"023320e4-5162-4756-a05b-2d501649b77a","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:36.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"023320e4-5162-4756-a05b-2d501649b77a","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:36.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"023320e4-5162-4756-a05b-2d501649b77a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:36.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"023320e4-5162-4756-a05b-2d501649b77a","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:36.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"023320e4-5162-4756-a05b-2d501649b77a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:36.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"023320e4-5162-4756-a05b-2d501649b77a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:36.337Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"023320e4-5162-4756-a05b-2d501649b77a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:36.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"023320e4-5162-4756-a05b-2d501649b77a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:36.337Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"023320e4-5162-4756-a05b-2d501649b77a","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:03:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:03:36.337Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"023320e4-5162-4756-a05b-2d501649b77a","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:03:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:03:36.337Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"023320e4-5162-4756-a05b-2d501649b77a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:36.337Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"023320e4-5162-4756-a05b-2d501649b77a","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:37.705Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2a11f599-824c-4049-b5ca-b175c22cf15c"}
{"level":"info","ts":"2026-09-18T10:03:37.705Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2a11f599-824c-4049-b5ca-b175c22cf15c"}
{"level":"info","ts":"2026-09-18T10:03:37.705Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2a11f599-824c-4049-b5ca-b175c22cf15c"}
{"level":"info","ts":"2026-09-18T10:03:37.705Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2a11f599-824c-4049-b5ca-b175c22cf15c","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:37.705Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2a11f599-824c-4049-b5ca-b175c22cf15c","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:37.705Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2a11f599-824c-4049-b5ca-b175c22cf15c","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:37.705Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2a11f599-824c-4049-b5ca-b175c22cf15c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:37.705Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2a11f599-824c-4049-b5ca-b175c22cf15c","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:37.705Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2a11f599-824c-4049-b5ca-b175c22cf15c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:37.705Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2a11f599-824c-4049-b5ca-b175c22cf15c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:37.705Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2a11f599-824c-4049-b5ca-b175c22cf15c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:37.705Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2a11f599-824c-4049-b5ca-b175c22cf15c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:37.705Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2a11f599-824c-4049-b5ca-b175c22cf15c","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:37.705Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2a11f599-824c-4049-b5ca-b175c22cf15c","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:37.706Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2a11f599-824c-4049-b5ca-b175c22cf15c","name":"long-running-5qfl8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.42","serviceIP":"10.133.0.42","podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.42","serviceIP":"10.133.0.42","podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:37Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:03:37.720Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"17097794-b930-405a-a4b7-211de78d2d24","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:37.720Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"e2b02d01-43b2-4879-a0fe-8fc4fd7b7292","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:37.720Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"e2b02d01-43b2-4879-a0fe-8fc4fd7b7292","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-09-18 10:03:20 +0000 UTC","Now":"2026-09-18T10:03:37.720Z","ShutdownTime":"2026-09-18T10:13:20.000Z"}
{"level":"info","ts":"2026-09-18T10:03:37.720Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"e2b02d01-43b2-4879-a0fe-8fc4fd7b7292","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-09-18T10:13:20.000Z","requeueAfter":584.279651206}
{"level":"info","ts":"2026-09-18T10:03:37.720Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"17097794-b930-405a-a4b7-211de78d2d24","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:03:37.720Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"17097794-b930-405a-a4b7-211de78d2d24","pod":"long-running-5qfl8-small-group-worker-tbqtj","ip":"10.132.0.47"}
{"level":"info","ts":"2026-09-18T10:03:37.720Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"17097794-b930-405a-a4b7-211de78d2d24","rayCluster":"long-running-5qfl8","podIPs":["10.133.0.42","10.132.0.47"]}
{"level":"info","ts":"2026-09-18T10:03:37.720Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"6599fd41-a369-4e0c-a001-a19069b9c9e8","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:37.731Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"6599fd41-a369-4e0c-a001-a19069b9c9e8","name":"long-running-5qfl8-head"}
{"level":"info","ts":"2026-09-18T10:03:37.743Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"6599fd41-a369-4e0c-a001-a19069b9c9e8","name":"long-running-5qfl8-workers"}
{"level":"info","ts":"2026-09-18T10:03:37.743Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"6599fd41-a369-4e0c-a001-a19069b9c9e8","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:39.313Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4619cb08-fd63-47b4-845b-217bd48d18a8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:39.313Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4619cb08-fd63-47b4-845b-217bd48d18a8","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:39.313Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4619cb08-fd63-47b4-845b-217bd48d18a8","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:39.313Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4619cb08-fd63-47b4-845b-217bd48d18a8"}
{"level":"info","ts":"2026-09-18T10:03:39.313Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4619cb08-fd63-47b4-845b-217bd48d18a8","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:39.313Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4619cb08-fd63-47b4-845b-217bd48d18a8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:39.720Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d5c501e0-1869-4d18-8e14-fe05e3303663"}
{"level":"info","ts":"2026-09-18T10:03:39.720Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d5c501e0-1869-4d18-8e14-fe05e3303663"}
{"level":"info","ts":"2026-09-18T10:03:39.720Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d5c501e0-1869-4d18-8e14-fe05e3303663"}
{"level":"info","ts":"2026-09-18T10:03:39.720Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d5c501e0-1869-4d18-8e14-fe05e3303663","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:39.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d5c501e0-1869-4d18-8e14-fe05e3303663","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:39.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d5c501e0-1869-4d18-8e14-fe05e3303663","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:39.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d5c501e0-1869-4d18-8e14-fe05e3303663","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:39.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d5c501e0-1869-4d18-8e14-fe05e3303663","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:39.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d5c501e0-1869-4d18-8e14-fe05e3303663","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:39.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d5c501e0-1869-4d18-8e14-fe05e3303663","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:39.721Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d5c501e0-1869-4d18-8e14-fe05e3303663","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:39.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d5c501e0-1869-4d18-8e14-fe05e3303663","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:39.721Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d5c501e0-1869-4d18-8e14-fe05e3303663","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:39.721Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d5c501e0-1869-4d18-8e14-fe05e3303663","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:39.721Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d5c501e0-1869-4d18-8e14-fe05e3303663","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:39.721Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d5c501e0-1869-4d18-8e14-fe05e3303663","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:42.314Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"020aabef-3bc9-4749-9844-b5d6b3b27ff0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:42.314Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"020aabef-3bc9-4749-9844-b5d6b3b27ff0","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:42.314Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"020aabef-3bc9-4749-9844-b5d6b3b27ff0","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:42.314Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"020aabef-3bc9-4749-9844-b5d6b3b27ff0"}
{"level":"info","ts":"2026-09-18T10:03:42.314Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"020aabef-3bc9-4749-9844-b5d6b3b27ff0","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:42.314Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"020aabef-3bc9-4749-9844-b5d6b3b27ff0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:45.314Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4d96b01a-d460-48aa-ab47-feaed01c2246","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:45.314Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4d96b01a-d460-48aa-ab47-feaed01c2246","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:45.314Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4d96b01a-d460-48aa-ab47-feaed01c2246","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:45.315Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4d96b01a-d460-48aa-ab47-feaed01c2246"}
{"level":"info","ts":"2026-09-18T10:03:45.315Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4d96b01a-d460-48aa-ab47-feaed01c2246","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:45.315Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4d96b01a-d460-48aa-ab47-feaed01c2246","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:45.349Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"70320998-b474-42e3-a2ef-d2c4773a6598"}
{"level":"info","ts":"2026-09-18T10:03:45.349Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"70320998-b474-42e3-a2ef-d2c4773a6598"}
{"level":"info","ts":"2026-09-18T10:03:45.349Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"70320998-b474-42e3-a2ef-d2c4773a6598"}
{"level":"info","ts":"2026-09-18T10:03:45.349Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"70320998-b474-42e3-a2ef-d2c4773a6598","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:45.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"70320998-b474-42e3-a2ef-d2c4773a6598","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:45.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"70320998-b474-42e3-a2ef-d2c4773a6598","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:45.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"70320998-b474-42e3-a2ef-d2c4773a6598","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:45.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"70320998-b474-42e3-a2ef-d2c4773a6598","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:45.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"70320998-b474-42e3-a2ef-d2c4773a6598","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:45.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"70320998-b474-42e3-a2ef-d2c4773a6598","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:45.349Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"70320998-b474-42e3-a2ef-d2c4773a6598","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:45.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"70320998-b474-42e3-a2ef-d2c4773a6598","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:45.349Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"70320998-b474-42e3-a2ef-d2c4773a6598","podName":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:45.349Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"70320998-b474-42e3-a2ef-d2c4773a6598","podName":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:45.349Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"70320998-b474-42e3-a2ef-d2c4773a6598","name":"delete-head-after-submit-r4xnr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:03:45.373Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"17f16f94-f670-409e-a129-b91d60dd02b7","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:45.373Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"4b925b3e-ae8d-45d6-af0d-9f7d970fa130","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:45.373Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"4b925b3e-ae8d-45d6-af0d-9f7d970fa130","pod":"delete-head-after-submit-r4xnr-head-kw54w","ip":"10.132.0.48"}
{"level":"info","ts":"2026-09-18T10:03:45.373Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"4b925b3e-ae8d-45d6-af0d-9f7d970fa130","pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","ip":"10.133.0.43"}
{"level":"info","ts":"2026-09-18T10:03:45.373Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"4b925b3e-ae8d-45d6-af0d-9f7d970fa130","rayCluster":"delete-head-after-submit-r4xnr","podIPs":["10.132.0.48","10.133.0.43"]}
{"level":"info","ts":"2026-09-18T10:03:45.373Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"92898bd1-3cb8-4660-8a2c-32151c8ad807","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:45.373Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"92898bd1-3cb8-4660-8a2c-32151c8ad807","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:45.373Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"92898bd1-3cb8-4660-8a2c-32151c8ad807","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:45.373Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"92898bd1-3cb8-4660-8a2c-32151c8ad807"}
{"level":"info","ts":"2026-09-18T10:03:45.373Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"92898bd1-3cb8-4660-8a2c-32151c8ad807","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:45.373Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"92898bd1-3cb8-4660-8a2c-32151c8ad807","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:45.373Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"92898bd1-3cb8-4660-8a2c-32151c8ad807","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:03:45.385Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"17f16f94-f670-409e-a129-b91d60dd02b7","name":"delete-head-after-submit-r4xnr-head"}
{"level":"info","ts":"2026-09-18T10:03:45.393Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4689e3ce-6cbe-4804-8ec5-f378bd988444","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:45.393Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4689e3ce-6cbe-4804-8ec5-f378bd988444","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:45.393Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4689e3ce-6cbe-4804-8ec5-f378bd988444","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:45.394Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4689e3ce-6cbe-4804-8ec5-f378bd988444"}
{"level":"info","ts":"2026-09-18T10:03:45.394Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4689e3ce-6cbe-4804-8ec5-f378bd988444","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:45.394Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4689e3ce-6cbe-4804-8ec5-f378bd988444","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:45.408Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"17f16f94-f670-409e-a129-b91d60dd02b7","name":"delete-head-after-submit-r4xnr-workers"}
{"level":"info","ts":"2026-09-18T10:03:45.408Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"17f16f94-f670-409e-a129-b91d60dd02b7","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:45.577Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"e721c3e9-5af8-4331-947f-4eb23847f082","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:03:45.577Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"e721c3e9-5af8-4331-947f-4eb23847f082","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:03:45.577Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"e721c3e9-5af8-4331-947f-4eb23847f082","pod":"long-running-5qfl8-small-group-worker-tbqtj","ip":"10.132.0.47"}
{"level":"info","ts":"2026-09-18T10:03:45.577Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"e721c3e9-5af8-4331-947f-4eb23847f082","rayCluster":"long-running-5qfl8","podIPs":["10.133.0.42","10.132.0.47"]}
{"level":"info","ts":"2026-09-18T10:03:47.375Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7da06159-9c77-4157-a98b-abfa5f0397ac"}
{"level":"info","ts":"2026-09-18T10:03:47.375Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7da06159-9c77-4157-a98b-abfa5f0397ac"}
{"level":"info","ts":"2026-09-18T10:03:47.375Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7da06159-9c77-4157-a98b-abfa5f0397ac"}
{"level":"info","ts":"2026-09-18T10:03:47.375Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7da06159-9c77-4157-a98b-abfa5f0397ac","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:03:47.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7da06159-9c77-4157-a98b-abfa5f0397ac","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:03:47.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7da06159-9c77-4157-a98b-abfa5f0397ac","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:47.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7da06159-9c77-4157-a98b-abfa5f0397ac","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:03:47.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7da06159-9c77-4157-a98b-abfa5f0397ac","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:03:47.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7da06159-9c77-4157-a98b-abfa5f0397ac","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:47.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7da06159-9c77-4157-a98b-abfa5f0397ac","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:03:47.375Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7da06159-9c77-4157-a98b-abfa5f0397ac","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:03:47.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7da06159-9c77-4157-a98b-abfa5f0397ac","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:03:47.375Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7da06159-9c77-4157-a98b-abfa5f0397ac","podName":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:47.375Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7da06159-9c77-4157-a98b-abfa5f0397ac","podName":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:03:47.375Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7da06159-9c77-4157-a98b-abfa5f0397ac","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:03:47.375Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7da06159-9c77-4157-a98b-abfa5f0397ac","seconds":300}
{"level":"info","ts":"2026-09-18T10:03:48.315Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"903aec2d-9d57-4e23-8b0e-9b9c6043b182","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:48.315Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"903aec2d-9d57-4e23-8b0e-9b9c6043b182","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:48.315Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"903aec2d-9d57-4e23-8b0e-9b9c6043b182","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:48.315Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"903aec2d-9d57-4e23-8b0e-9b9c6043b182"}
{"level":"info","ts":"2026-09-18T10:03:48.315Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"903aec2d-9d57-4e23-8b0e-9b9c6043b182","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:48.315Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"903aec2d-9d57-4e23-8b0e-9b9c6043b182","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:51.316Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"de304def-399a-40a1-91b2-c11c861089db","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:51.316Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"de304def-399a-40a1-91b2-c11c861089db","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:51.316Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"de304def-399a-40a1-91b2-c11c861089db","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:51.316Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"de304def-399a-40a1-91b2-c11c861089db"}
{"level":"info","ts":"2026-09-18T10:03:51.316Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"de304def-399a-40a1-91b2-c11c861089db","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:51.316Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"de304def-399a-40a1-91b2-c11c861089db","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:51.504Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"66c86a9f-ce1f-493f-b6f3-cbf0992bd20c","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:51.504Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"66c86a9f-ce1f-493f-b6f3-cbf0992bd20c","pod":"delete-head-after-submit-r4xnr-head-kw54w","ip":"10.132.0.48"}
{"level":"info","ts":"2026-09-18T10:03:51.504Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"66c86a9f-ce1f-493f-b6f3-cbf0992bd20c","pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","ip":"10.133.0.43"}
{"level":"info","ts":"2026-09-18T10:03:51.504Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"66c86a9f-ce1f-493f-b6f3-cbf0992bd20c","rayCluster":"delete-head-after-submit-r4xnr","podIPs":["10.132.0.48","10.133.0.43"]}
{"level":"info","ts":"2026-09-18T10:03:54.316Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a1fe27a3-e334-479c-a2d9-a56eec4020b9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:54.316Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a1fe27a3-e334-479c-a2d9-a56eec4020b9","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:54.316Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a1fe27a3-e334-479c-a2d9-a56eec4020b9","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:54.317Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a1fe27a3-e334-479c-a2d9-a56eec4020b9"}
{"level":"info","ts":"2026-09-18T10:03:54.317Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a1fe27a3-e334-479c-a2d9-a56eec4020b9","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:54.317Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a1fe27a3-e334-479c-a2d9-a56eec4020b9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:03:57.318Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"e7532178-3561-4065-8dd8-1fc9cdad30d3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:03:57.318Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"e7532178-3561-4065-8dd8-1fc9cdad30d3","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:03:57.318Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"e7532178-3561-4065-8dd8-1fc9cdad30d3","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:03:57.318Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"e7532178-3561-4065-8dd8-1fc9cdad30d3"}
{"level":"info","ts":"2026-09-18T10:03:57.318Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"e7532178-3561-4065-8dd8-1fc9cdad30d3","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:03:57.318Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"e7532178-3561-4065-8dd8-1fc9cdad30d3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:00.318Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"bf7c6805-30a3-4361-a9d0-fe15e3581403","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:00.318Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"bf7c6805-30a3-4361-a9d0-fe15e3581403","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:00.318Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"bf7c6805-30a3-4361-a9d0-fe15e3581403","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:00.319Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"bf7c6805-30a3-4361-a9d0-fe15e3581403"}
{"level":"info","ts":"2026-09-18T10:04:00.319Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"bf7c6805-30a3-4361-a9d0-fe15e3581403","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:00.319Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"bf7c6805-30a3-4361-a9d0-fe15e3581403","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:03.320Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"59d871f1-7f84-4469-9a6c-2255091794cf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:03.320Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"59d871f1-7f84-4469-9a6c-2255091794cf","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:03.320Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"59d871f1-7f84-4469-9a6c-2255091794cf","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:03.320Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"59d871f1-7f84-4469-9a6c-2255091794cf"}
{"level":"info","ts":"2026-09-18T10:04:03.320Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"59d871f1-7f84-4469-9a6c-2255091794cf","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:03.320Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"59d871f1-7f84-4469-9a6c-2255091794cf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:06.321Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"e8f30dbf-1d2c-419b-8533-ab6bedf2c6c8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:06.321Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"e8f30dbf-1d2c-419b-8533-ab6bedf2c6c8","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:06.321Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"e8f30dbf-1d2c-419b-8533-ab6bedf2c6c8","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:06.321Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"e8f30dbf-1d2c-419b-8533-ab6bedf2c6c8"}
{"level":"info","ts":"2026-09-18T10:04:06.321Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"e8f30dbf-1d2c-419b-8533-ab6bedf2c6c8","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:06.321Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"e8f30dbf-1d2c-419b-8533-ab6bedf2c6c8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:09.322Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"cf71dfcd-f351-4f51-98fc-e8b66e851941","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:09.322Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"cf71dfcd-f351-4f51-98fc-e8b66e851941","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:09.322Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"cf71dfcd-f351-4f51-98fc-e8b66e851941","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:09.322Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"cf71dfcd-f351-4f51-98fc-e8b66e851941"}
{"level":"info","ts":"2026-09-18T10:04:09.322Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"cf71dfcd-f351-4f51-98fc-e8b66e851941","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:09.322Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"cf71dfcd-f351-4f51-98fc-e8b66e851941","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:12.323Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"fceec532-8df0-422d-a257-30a79b630d2b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:12.323Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"fceec532-8df0-422d-a257-30a79b630d2b","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:12.323Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"fceec532-8df0-422d-a257-30a79b630d2b","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:12.324Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"fceec532-8df0-422d-a257-30a79b630d2b"}
{"level":"info","ts":"2026-09-18T10:04:12.324Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"fceec532-8df0-422d-a257-30a79b630d2b","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:12.324Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"fceec532-8df0-422d-a257-30a79b630d2b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:15.324Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"78d47914-5d52-4fc9-9459-ea06da16b9e8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:15.324Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"78d47914-5d52-4fc9-9459-ea06da16b9e8","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:15.324Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"78d47914-5d52-4fc9-9459-ea06da16b9e8","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:15.324Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"78d47914-5d52-4fc9-9459-ea06da16b9e8"}
{"level":"info","ts":"2026-09-18T10:04:15.324Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"78d47914-5d52-4fc9-9459-ea06da16b9e8","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:15.324Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"78d47914-5d52-4fc9-9459-ea06da16b9e8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:18.325Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a92f4923-9dd2-4929-a164-46c66350b941","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:18.325Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a92f4923-9dd2-4929-a164-46c66350b941","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:18.325Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a92f4923-9dd2-4929-a164-46c66350b941","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:18.325Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a92f4923-9dd2-4929-a164-46c66350b941"}
{"level":"info","ts":"2026-09-18T10:04:18.325Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a92f4923-9dd2-4929-a164-46c66350b941","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:18.325Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a92f4923-9dd2-4929-a164-46c66350b941","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:21.326Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"c958c852-630b-4374-9e0d-810177dc852b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:21.326Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"c958c852-630b-4374-9e0d-810177dc852b","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:21.326Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"c958c852-630b-4374-9e0d-810177dc852b","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:21.326Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"c958c852-630b-4374-9e0d-810177dc852b"}
{"level":"info","ts":"2026-09-18T10:04:21.326Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"c958c852-630b-4374-9e0d-810177dc852b","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:21.326Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"c958c852-630b-4374-9e0d-810177dc852b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:24.327Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"634b0a92-c693-497f-85ed-6154fb4255dc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:24.327Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"634b0a92-c693-497f-85ed-6154fb4255dc","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:24.327Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"634b0a92-c693-497f-85ed-6154fb4255dc","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:24.327Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"634b0a92-c693-497f-85ed-6154fb4255dc"}
{"level":"info","ts":"2026-09-18T10:04:24.327Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"634b0a92-c693-497f-85ed-6154fb4255dc","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:24.327Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"634b0a92-c693-497f-85ed-6154fb4255dc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:27.328Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4b4d611b-0bfd-4895-8e02-38cbf7ba45ea","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:27.328Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4b4d611b-0bfd-4895-8e02-38cbf7ba45ea","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:27.328Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4b4d611b-0bfd-4895-8e02-38cbf7ba45ea","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:27.329Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4b4d611b-0bfd-4895-8e02-38cbf7ba45ea"}
{"level":"info","ts":"2026-09-18T10:04:27.329Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4b4d611b-0bfd-4895-8e02-38cbf7ba45ea","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:27.329Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4b4d611b-0bfd-4895-8e02-38cbf7ba45ea","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:30.329Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"914c07de-f4f5-48a9-97ad-ba5454cf5b83","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:30.329Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"914c07de-f4f5-48a9-97ad-ba5454cf5b83","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:30.329Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"914c07de-f4f5-48a9-97ad-ba5454cf5b83","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:30.329Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"914c07de-f4f5-48a9-97ad-ba5454cf5b83"}
{"level":"info","ts":"2026-09-18T10:04:30.329Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"914c07de-f4f5-48a9-97ad-ba5454cf5b83","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:30.329Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"914c07de-f4f5-48a9-97ad-ba5454cf5b83","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:33.329Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2584a278-5e83-4306-b698-0f07647a5232","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:33.329Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2584a278-5e83-4306-b698-0f07647a5232","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:33.330Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2584a278-5e83-4306-b698-0f07647a5232","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:33.330Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2584a278-5e83-4306-b698-0f07647a5232"}
{"level":"info","ts":"2026-09-18T10:04:33.330Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2584a278-5e83-4306-b698-0f07647a5232","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:33.330Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2584a278-5e83-4306-b698-0f07647a5232","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:36.331Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"03555091-8baa-4d3f-a9d5-9d0b190332c6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:36.331Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"03555091-8baa-4d3f-a9d5-9d0b190332c6","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:36.331Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"03555091-8baa-4d3f-a9d5-9d0b190332c6","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:36.331Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"03555091-8baa-4d3f-a9d5-9d0b190332c6"}
{"level":"info","ts":"2026-09-18T10:04:36.331Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"03555091-8baa-4d3f-a9d5-9d0b190332c6","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:36.331Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"03555091-8baa-4d3f-a9d5-9d0b190332c6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:39.331Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"96a26139-b068-4f73-b412-0a308306cde7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:39.331Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"96a26139-b068-4f73-b412-0a308306cde7","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:39.331Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"96a26139-b068-4f73-b412-0a308306cde7","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:39.332Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"96a26139-b068-4f73-b412-0a308306cde7"}
{"level":"info","ts":"2026-09-18T10:04:39.332Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"96a26139-b068-4f73-b412-0a308306cde7","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:39.332Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"96a26139-b068-4f73-b412-0a308306cde7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:42.332Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"de0a61a4-cdcb-45bc-b18f-5f94b5f9be33","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:42.332Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"de0a61a4-cdcb-45bc-b18f-5f94b5f9be33","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:42.332Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"de0a61a4-cdcb-45bc-b18f-5f94b5f9be33","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:42.332Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"de0a61a4-cdcb-45bc-b18f-5f94b5f9be33"}
{"level":"info","ts":"2026-09-18T10:04:42.332Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"de0a61a4-cdcb-45bc-b18f-5f94b5f9be33","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:42.332Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"de0a61a4-cdcb-45bc-b18f-5f94b5f9be33","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:43.002Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e5176da6-054e-4921-8db1-ef24ebed30e5"}
{"level":"info","ts":"2026-09-18T10:04:43.002Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e5176da6-054e-4921-8db1-ef24ebed30e5"}
{"level":"info","ts":"2026-09-18T10:04:43.002Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e5176da6-054e-4921-8db1-ef24ebed30e5"}
{"level":"info","ts":"2026-09-18T10:04:43.002Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e5176da6-054e-4921-8db1-ef24ebed30e5","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:04:43.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e5176da6-054e-4921-8db1-ef24ebed30e5","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:04:43.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e5176da6-054e-4921-8db1-ef24ebed30e5","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:04:43.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e5176da6-054e-4921-8db1-ef24ebed30e5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:04:43.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e5176da6-054e-4921-8db1-ef24ebed30e5","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:04:43.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e5176da6-054e-4921-8db1-ef24ebed30e5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:04:43.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e5176da6-054e-4921-8db1-ef24ebed30e5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:04:43.002Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e5176da6-054e-4921-8db1-ef24ebed30e5","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:04:43.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e5176da6-054e-4921-8db1-ef24ebed30e5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:04:43.002Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e5176da6-054e-4921-8db1-ef24ebed30e5","podName":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:04:43.002Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e5176da6-054e-4921-8db1-ef24ebed30e5","podName":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:04:43.003Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e5176da6-054e-4921-8db1-ef24ebed30e5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:04:43.003Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"e5176da6-054e-4921-8db1-ef24ebed30e5","seconds":300}
{"level":"info","ts":"2026-09-18T10:04:44.006Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"bcc85b0e-fd1f-4d9d-b42a-65cad1d8c188"}
{"level":"info","ts":"2026-09-18T10:04:44.006Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"bcc85b0e-fd1f-4d9d-b42a-65cad1d8c188"}
{"level":"info","ts":"2026-09-18T10:04:44.006Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"bcc85b0e-fd1f-4d9d-b42a-65cad1d8c188"}
{"level":"info","ts":"2026-09-18T10:04:44.006Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"bcc85b0e-fd1f-4d9d-b42a-65cad1d8c188","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:04:44.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"bcc85b0e-fd1f-4d9d-b42a-65cad1d8c188","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:04:44.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"bcc85b0e-fd1f-4d9d-b42a-65cad1d8c188","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:04:44.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"bcc85b0e-fd1f-4d9d-b42a-65cad1d8c188","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:04:44.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"bcc85b0e-fd1f-4d9d-b42a-65cad1d8c188","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:04:44.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"bcc85b0e-fd1f-4d9d-b42a-65cad1d8c188","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:04:44.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"bcc85b0e-fd1f-4d9d-b42a-65cad1d8c188","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:04:44.006Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"bcc85b0e-fd1f-4d9d-b42a-65cad1d8c188","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:04:44.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"bcc85b0e-fd1f-4d9d-b42a-65cad1d8c188","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:04:44.006Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"bcc85b0e-fd1f-4d9d-b42a-65cad1d8c188","podName":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:04:44.006Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"bcc85b0e-fd1f-4d9d-b42a-65cad1d8c188","podName":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:04:44.006Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"bcc85b0e-fd1f-4d9d-b42a-65cad1d8c188","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:04:44.006Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"bcc85b0e-fd1f-4d9d-b42a-65cad1d8c188","seconds":300}
{"level":"info","ts":"2026-09-18T10:04:45.334Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"49f45cfa-b92e-4caf-b5f3-c3164fbd2a45","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:45.334Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"49f45cfa-b92e-4caf-b5f3-c3164fbd2a45","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:45.334Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"49f45cfa-b92e-4caf-b5f3-c3164fbd2a45","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:45.334Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"49f45cfa-b92e-4caf-b5f3-c3164fbd2a45"}
{"level":"info","ts":"2026-09-18T10:04:45.334Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"49f45cfa-b92e-4caf-b5f3-c3164fbd2a45","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:45.334Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"49f45cfa-b92e-4caf-b5f3-c3164fbd2a45","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:45.578Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"f5458b95-0542-4b26-879f-558b38b38c07","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:04:45.578Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"f5458b95-0542-4b26-879f-558b38b38c07","pod":"long-running-5qfl8-small-group-worker-tbqtj","ip":"10.132.0.47"}
{"level":"info","ts":"2026-09-18T10:04:45.578Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"f5458b95-0542-4b26-879f-558b38b38c07","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:04:45.578Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"f5458b95-0542-4b26-879f-558b38b38c07","rayCluster":"long-running-5qfl8","podIPs":["10.132.0.47","10.133.0.42"]}
{"level":"info","ts":"2026-09-18T10:04:48.335Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"1b822cf9-e1a2-4961-8480-bf6c2cc24f9a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:48.335Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"1b822cf9-e1a2-4961-8480-bf6c2cc24f9a","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:48.335Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"1b822cf9-e1a2-4961-8480-bf6c2cc24f9a","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:48.335Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"1b822cf9-e1a2-4961-8480-bf6c2cc24f9a"}
{"level":"info","ts":"2026-09-18T10:04:48.335Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"1b822cf9-e1a2-4961-8480-bf6c2cc24f9a","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:48.335Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"1b822cf9-e1a2-4961-8480-bf6c2cc24f9a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:51.335Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ffa24b8a-1faf-423d-9a4c-c86e4537b400","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:51.335Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ffa24b8a-1faf-423d-9a4c-c86e4537b400","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:51.335Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ffa24b8a-1faf-423d-9a4c-c86e4537b400","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:51.336Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ffa24b8a-1faf-423d-9a4c-c86e4537b400"}
{"level":"info","ts":"2026-09-18T10:04:51.336Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ffa24b8a-1faf-423d-9a4c-c86e4537b400","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:51.336Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ffa24b8a-1faf-423d-9a4c-c86e4537b400","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:51.505Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"12baf253-2800-4c48-a32e-78b9610f85be","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:51.505Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"12baf253-2800-4c48-a32e-78b9610f85be","pod":"delete-head-after-submit-r4xnr-head-kw54w","ip":"10.132.0.48"}
{"level":"info","ts":"2026-09-18T10:04:51.505Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"12baf253-2800-4c48-a32e-78b9610f85be","pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","ip":"10.133.0.43"}
{"level":"info","ts":"2026-09-18T10:04:51.505Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"12baf253-2800-4c48-a32e-78b9610f85be","rayCluster":"delete-head-after-submit-r4xnr","podIPs":["10.132.0.48","10.133.0.43"]}
{"level":"info","ts":"2026-09-18T10:04:53.061Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2a8b046f-a32c-4c78-89ac-de395aa32cc2"}
{"level":"info","ts":"2026-09-18T10:04:53.061Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2a8b046f-a32c-4c78-89ac-de395aa32cc2"}
{"level":"info","ts":"2026-09-18T10:04:53.061Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2a8b046f-a32c-4c78-89ac-de395aa32cc2"}
{"level":"info","ts":"2026-09-18T10:04:53.061Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2a8b046f-a32c-4c78-89ac-de395aa32cc2","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:04:53.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2a8b046f-a32c-4c78-89ac-de395aa32cc2","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:04:53.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2a8b046f-a32c-4c78-89ac-de395aa32cc2","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:04:53.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2a8b046f-a32c-4c78-89ac-de395aa32cc2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:04:53.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2a8b046f-a32c-4c78-89ac-de395aa32cc2","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:04:53.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2a8b046f-a32c-4c78-89ac-de395aa32cc2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:04:53.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2a8b046f-a32c-4c78-89ac-de395aa32cc2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:04:53.061Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2a8b046f-a32c-4c78-89ac-de395aa32cc2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:04:53.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2a8b046f-a32c-4c78-89ac-de395aa32cc2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:04:53.062Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2a8b046f-a32c-4c78-89ac-de395aa32cc2","podName":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:04:53.062Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2a8b046f-a32c-4c78-89ac-de395aa32cc2","podName":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:04:53.062Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2a8b046f-a32c-4c78-89ac-de395aa32cc2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:04:53.062Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2a8b046f-a32c-4c78-89ac-de395aa32cc2","seconds":300}
{"level":"info","ts":"2026-09-18T10:04:54.073Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4ff504e7-d4f8-4317-acb2-77019f8cb4d0"}
{"level":"info","ts":"2026-09-18T10:04:54.073Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4ff504e7-d4f8-4317-acb2-77019f8cb4d0"}
{"level":"info","ts":"2026-09-18T10:04:54.073Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4ff504e7-d4f8-4317-acb2-77019f8cb4d0"}
{"level":"info","ts":"2026-09-18T10:04:54.073Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4ff504e7-d4f8-4317-acb2-77019f8cb4d0","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:04:54.073Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4ff504e7-d4f8-4317-acb2-77019f8cb4d0","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:04:54.073Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4ff504e7-d4f8-4317-acb2-77019f8cb4d0","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:04:54.073Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4ff504e7-d4f8-4317-acb2-77019f8cb4d0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:04:54.073Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4ff504e7-d4f8-4317-acb2-77019f8cb4d0","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:04:54.073Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4ff504e7-d4f8-4317-acb2-77019f8cb4d0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:04:54.073Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4ff504e7-d4f8-4317-acb2-77019f8cb4d0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:04:54.073Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4ff504e7-d4f8-4317-acb2-77019f8cb4d0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:04:54.073Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4ff504e7-d4f8-4317-acb2-77019f8cb4d0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:04:54.073Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4ff504e7-d4f8-4317-acb2-77019f8cb4d0","podName":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:03:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:04:54.073Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4ff504e7-d4f8-4317-acb2-77019f8cb4d0","podName":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:03:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:04:54.073Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4ff504e7-d4f8-4317-acb2-77019f8cb4d0","name":"delete-head-after-submit-r4xnr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:04:54.093Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"27962181-5737-4886-b436-5341033e977a","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:54.093Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b5a4be0c-25da-46e8-94a4-31178399d2db","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:54.093Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b5a4be0c-25da-46e8-94a4-31178399d2db","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:54.093Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b5a4be0c-25da-46e8-94a4-31178399d2db","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:54.094Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b5a4be0c-25da-46e8-94a4-31178399d2db"}
{"level":"info","ts":"2026-09-18T10:04:54.094Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b5a4be0c-25da-46e8-94a4-31178399d2db","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:54.094Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b5a4be0c-25da-46e8-94a4-31178399d2db","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:54.094Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b5a4be0c-25da-46e8-94a4-31178399d2db","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:04:54.094Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"45f662b3-5cc7-40e5-8acd-d661db05e346","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:54.094Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"45f662b3-5cc7-40e5-8acd-d661db05e346","pod":"delete-head-after-submit-r4xnr-head-kw54w","ip":"10.132.0.48"}
{"level":"info","ts":"2026-09-18T10:04:54.094Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"45f662b3-5cc7-40e5-8acd-d661db05e346","pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","ip":"10.133.0.43"}
{"level":"info","ts":"2026-09-18T10:04:54.094Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"45f662b3-5cc7-40e5-8acd-d661db05e346","rayCluster":"delete-head-after-submit-r4xnr","podIPs":["10.132.0.48","10.133.0.43"]}
{"level":"info","ts":"2026-09-18T10:04:54.166Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"8d30f538-dd30-41cf-a79b-3a046e109973","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:54.166Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"8d30f538-dd30-41cf-a79b-3a046e109973","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:54.166Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"8d30f538-dd30-41cf-a79b-3a046e109973","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:54.166Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"8d30f538-dd30-41cf-a79b-3a046e109973"}
{"level":"info","ts":"2026-09-18T10:04:54.166Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"8d30f538-dd30-41cf-a79b-3a046e109973","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:54.166Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"8d30f538-dd30-41cf-a79b-3a046e109973","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:54.172Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"27962181-5737-4886-b436-5341033e977a","name":"delete-head-after-submit-r4xnr-head"}
{"level":"info","ts":"2026-09-18T10:04:54.193Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"27962181-5737-4886-b436-5341033e977a","name":"delete-head-after-submit-r4xnr-workers"}
{"level":"info","ts":"2026-09-18T10:04:54.193Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"27962181-5737-4886-b436-5341033e977a","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:54.337Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"cc4b3f5e-3b08-44ef-b174-7a8fef4a774a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:54.337Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"cc4b3f5e-3b08-44ef-b174-7a8fef4a774a","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:54.337Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"cc4b3f5e-3b08-44ef-b174-7a8fef4a774a","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:54.337Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"cc4b3f5e-3b08-44ef-b174-7a8fef4a774a"}
{"level":"info","ts":"2026-09-18T10:04:54.337Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"cc4b3f5e-3b08-44ef-b174-7a8fef4a774a","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:54.337Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"cc4b3f5e-3b08-44ef-b174-7a8fef4a774a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:04:54.704Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1fc130a6-4d9d-45c7-9d5d-01114a41e451"}
{"level":"info","ts":"2026-09-18T10:04:54.704Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1fc130a6-4d9d-45c7-9d5d-01114a41e451"}
{"level":"info","ts":"2026-09-18T10:04:54.704Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1fc130a6-4d9d-45c7-9d5d-01114a41e451"}
{"level":"info","ts":"2026-09-18T10:04:54.704Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1fc130a6-4d9d-45c7-9d5d-01114a41e451","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:04:54.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1fc130a6-4d9d-45c7-9d5d-01114a41e451","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:04:54.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1fc130a6-4d9d-45c7-9d5d-01114a41e451","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:04:54.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1fc130a6-4d9d-45c7-9d5d-01114a41e451","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:04:54.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1fc130a6-4d9d-45c7-9d5d-01114a41e451","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:04:54.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1fc130a6-4d9d-45c7-9d5d-01114a41e451","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:04:54.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1fc130a6-4d9d-45c7-9d5d-01114a41e451","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:04:54.704Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1fc130a6-4d9d-45c7-9d5d-01114a41e451","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:04:54.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1fc130a6-4d9d-45c7-9d5d-01114a41e451","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:04:54.705Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1fc130a6-4d9d-45c7-9d5d-01114a41e451","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:04:54.705Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1fc130a6-4d9d-45c7-9d5d-01114a41e451","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:04:54.705Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1fc130a6-4d9d-45c7-9d5d-01114a41e451","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:04:54.705Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"1fc130a6-4d9d-45c7-9d5d-01114a41e451","seconds":300}
{"level":"info","ts":"2026-09-18T10:04:55.715Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4017f42a-eca5-437e-8477-4c666789d4ea"}
{"level":"info","ts":"2026-09-18T10:04:55.715Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4017f42a-eca5-437e-8477-4c666789d4ea"}
{"level":"info","ts":"2026-09-18T10:04:55.715Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4017f42a-eca5-437e-8477-4c666789d4ea"}
{"level":"info","ts":"2026-09-18T10:04:55.715Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4017f42a-eca5-437e-8477-4c666789d4ea","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:04:55.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4017f42a-eca5-437e-8477-4c666789d4ea","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:04:55.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4017f42a-eca5-437e-8477-4c666789d4ea","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:04:55.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4017f42a-eca5-437e-8477-4c666789d4ea","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:04:55.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4017f42a-eca5-437e-8477-4c666789d4ea","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:04:55.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4017f42a-eca5-437e-8477-4c666789d4ea","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:04:55.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4017f42a-eca5-437e-8477-4c666789d4ea","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:04:55.715Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4017f42a-eca5-437e-8477-4c666789d4ea","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:04:55.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4017f42a-eca5-437e-8477-4c666789d4ea","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:04:55.715Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4017f42a-eca5-437e-8477-4c666789d4ea","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:04:55.715Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4017f42a-eca5-437e-8477-4c666789d4ea","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:04:55.715Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4017f42a-eca5-437e-8477-4c666789d4ea","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:04:55.715Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"4017f42a-eca5-437e-8477-4c666789d4ea","seconds":300}
{"level":"info","ts":"2026-09-18T10:04:56.094Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6ad7e29d-b46b-440a-b1d7-8f7f06da8e7f"}
{"level":"info","ts":"2026-09-18T10:04:56.094Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6ad7e29d-b46b-440a-b1d7-8f7f06da8e7f"}
{"level":"info","ts":"2026-09-18T10:04:56.094Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6ad7e29d-b46b-440a-b1d7-8f7f06da8e7f"}
{"level":"info","ts":"2026-09-18T10:04:56.094Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6ad7e29d-b46b-440a-b1d7-8f7f06da8e7f","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:04:56.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6ad7e29d-b46b-440a-b1d7-8f7f06da8e7f","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:04:56.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6ad7e29d-b46b-440a-b1d7-8f7f06da8e7f","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:04:56.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6ad7e29d-b46b-440a-b1d7-8f7f06da8e7f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:04:56.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6ad7e29d-b46b-440a-b1d7-8f7f06da8e7f","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:04:56.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6ad7e29d-b46b-440a-b1d7-8f7f06da8e7f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:04:56.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6ad7e29d-b46b-440a-b1d7-8f7f06da8e7f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:04:56.094Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6ad7e29d-b46b-440a-b1d7-8f7f06da8e7f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:04:56.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6ad7e29d-b46b-440a-b1d7-8f7f06da8e7f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:04:56.095Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6ad7e29d-b46b-440a-b1d7-8f7f06da8e7f","podName":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:03:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:04:56.095Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6ad7e29d-b46b-440a-b1d7-8f7f06da8e7f","podName":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:03:26Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:04:56.095Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6ad7e29d-b46b-440a-b1d7-8f7f06da8e7f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:04:56.095Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6ad7e29d-b46b-440a-b1d7-8f7f06da8e7f","seconds":300}
{"level":"info","ts":"2026-09-18T10:04:57.337Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"6bdf1c07-877a-4786-99b4-32e45a76902d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:04:57.337Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"6bdf1c07-877a-4786-99b4-32e45a76902d","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:04:57.337Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"6bdf1c07-877a-4786-99b4-32e45a76902d","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:04:57.338Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"6bdf1c07-877a-4786-99b4-32e45a76902d"}
{"level":"info","ts":"2026-09-18T10:04:57.338Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"6bdf1c07-877a-4786-99b4-32e45a76902d","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:04:57.338Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"6bdf1c07-877a-4786-99b4-32e45a76902d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:05:00.338Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d4951841-1db3-4453-a677-0ca340a911a0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:00.338Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d4951841-1db3-4453-a677-0ca340a911a0","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:00.338Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d4951841-1db3-4453-a677-0ca340a911a0","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:00.338Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d4951841-1db3-4453-a677-0ca340a911a0"}
{"level":"info","ts":"2026-09-18T10:05:00.338Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d4951841-1db3-4453-a677-0ca340a911a0","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:05:00.338Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d4951841-1db3-4453-a677-0ca340a911a0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:05:03.338Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d80f4bd5-00dc-48a0-86dc-e405c0e96a7a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:03.338Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d80f4bd5-00dc-48a0-86dc-e405c0e96a7a","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:03.338Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d80f4bd5-00dc-48a0-86dc-e405c0e96a7a","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:03.339Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d80f4bd5-00dc-48a0-86dc-e405c0e96a7a"}
{"level":"info","ts":"2026-09-18T10:05:03.339Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d80f4bd5-00dc-48a0-86dc-e405c0e96a7a","RayCluster":"delete-head-after-submit-r4xnr","State":""}
{"level":"info","ts":"2026-09-18T10:05:03.339Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d80f4bd5-00dc-48a0-86dc-e405c0e96a7a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:05:05.098Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a1e4fae-485d-4f0e-835b-36b9d3d1f1fa"}
{"level":"info","ts":"2026-09-18T10:05:05.098Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a1e4fae-485d-4f0e-835b-36b9d3d1f1fa"}
{"level":"info","ts":"2026-09-18T10:05:05.098Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a1e4fae-485d-4f0e-835b-36b9d3d1f1fa"}
{"level":"info","ts":"2026-09-18T10:05:05.098Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a1e4fae-485d-4f0e-835b-36b9d3d1f1fa","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:05.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a1e4fae-485d-4f0e-835b-36b9d3d1f1fa","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:05.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a1e4fae-485d-4f0e-835b-36b9d3d1f1fa","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:05.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a1e4fae-485d-4f0e-835b-36b9d3d1f1fa","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:05.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a1e4fae-485d-4f0e-835b-36b9d3d1f1fa","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:05.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a1e4fae-485d-4f0e-835b-36b9d3d1f1fa","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:05.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a1e4fae-485d-4f0e-835b-36b9d3d1f1fa","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:05.099Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a1e4fae-485d-4f0e-835b-36b9d3d1f1fa","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:05.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a1e4fae-485d-4f0e-835b-36b9d3d1f1fa","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:05.099Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7a1e4fae-485d-4f0e-835b-36b9d3d1f1fa","name":"delete-head-after-submit-r4xnr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:05:05Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:05:05.123Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"33d52f65-319a-4030-88ce-5699d4705ba6","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:05.123Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b396544b-11c0-4d30-8e71-099a8ab6fb20","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:05.123Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b396544b-11c0-4d30-8e71-099a8ab6fb20","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:05.123Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b396544b-11c0-4d30-8e71-099a8ab6fb20","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:05.123Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"69650e0c-02a1-4176-8759-a76d4713b939","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:05.123Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b396544b-11c0-4d30-8e71-099a8ab6fb20"}
{"level":"info","ts":"2026-09-18T10:05:05.123Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"69650e0c-02a1-4176-8759-a76d4713b939","pod":"delete-head-after-submit-r4xnr-head-kw54w","ip":"10.132.0.48"}
{"level":"info","ts":"2026-09-18T10:05:05.123Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"69650e0c-02a1-4176-8759-a76d4713b939","pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","ip":"10.133.0.43"}
{"level":"info","ts":"2026-09-18T10:05:05.123Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"69650e0c-02a1-4176-8759-a76d4713b939","rayCluster":"delete-head-after-submit-r4xnr","podIPs":["10.132.0.48","10.133.0.43"]}
{"level":"info","ts":"2026-09-18T10:05:05.175Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b396544b-11c0-4d30-8e71-099a8ab6fb20","Kubernetes Job":"delete-head-after-submit"}
{"level":"info","ts":"2026-09-18T10:05:05.175Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b396544b-11c0-4d30-8e71-099a8ab6fb20","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","dashboardURL":"delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:05:05.175Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"b396544b-11c0-4d30-8e71-099a8ab6fb20","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:05:05.194Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"33d52f65-319a-4030-88ce-5699d4705ba6","name":"delete-head-after-submit-r4xnr-head"}
{"level":"info","ts":"2026-09-18T10:05:05.195Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9a904ec7-a0ce-4a34-b9b3-19e0d2fc2a11","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:05.195Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9a904ec7-a0ce-4a34-b9b3-19e0d2fc2a11","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:05.195Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9a904ec7-a0ce-4a34-b9b3-19e0d2fc2a11","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:05.196Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9a904ec7-a0ce-4a34-b9b3-19e0d2fc2a11"}
{"level":"info","ts":"2026-09-18T10:05:05.196Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9a904ec7-a0ce-4a34-b9b3-19e0d2fc2a11","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:05.237Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9a904ec7-a0ce-4a34-b9b3-19e0d2fc2a11","JobId":"delete-head-after-submit-rct8k","error":"Job delete-head-after-submit-rct8k does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:05.237Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9a904ec7-a0ce-4a34-b9b3-19e0d2fc2a11"}
{"level":"error","ts":"2026-09-18T10:05:05.237Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"9a904ec7-a0ce-4a34-b9b3-19e0d2fc2a11","error":"Job delete-head-after-submit-rct8k does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:05.237Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"13fe2235-6c73-45e6-87c6-b98c1e4faea8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:05.237Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"13fe2235-6c73-45e6-87c6-b98c1e4faea8","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:05.237Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"13fe2235-6c73-45e6-87c6-b98c1e4faea8","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:05.238Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"13fe2235-6c73-45e6-87c6-b98c1e4faea8"}
{"level":"info","ts":"2026-09-18T10:05:05.238Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"13fe2235-6c73-45e6-87c6-b98c1e4faea8","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:05.242Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"13fe2235-6c73-45e6-87c6-b98c1e4faea8","JobId":"delete-head-after-submit-rct8k","error":"Job delete-head-after-submit-rct8k does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:05.242Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"13fe2235-6c73-45e6-87c6-b98c1e4faea8"}
{"level":"error","ts":"2026-09-18T10:05:05.242Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"13fe2235-6c73-45e6-87c6-b98c1e4faea8","error":"Job delete-head-after-submit-rct8k does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:05.242Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"8d0b96d9-d52e-4ab1-9257-bd42f26c88cc","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:05.242Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"8d0b96d9-d52e-4ab1-9257-bd42f26c88cc","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:05.243Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"8d0b96d9-d52e-4ab1-9257-bd42f26c88cc","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:05.243Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"8d0b96d9-d52e-4ab1-9257-bd42f26c88cc"}
{"level":"info","ts":"2026-09-18T10:05:05.243Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"8d0b96d9-d52e-4ab1-9257-bd42f26c88cc","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:05.271Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"8d0b96d9-d52e-4ab1-9257-bd42f26c88cc","JobId":"delete-head-after-submit-rct8k","error":"Job delete-head-after-submit-rct8k does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:05.271Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"8d0b96d9-d52e-4ab1-9257-bd42f26c88cc"}
{"level":"error","ts":"2026-09-18T10:05:05.271Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"8d0b96d9-d52e-4ab1-9257-bd42f26c88cc","error":"Job delete-head-after-submit-rct8k does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:05.291Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"990e2cc7-d705-4156-a9e6-0c4a2cbe5181","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:05.291Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"990e2cc7-d705-4156-a9e6-0c4a2cbe5181","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:05.291Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"990e2cc7-d705-4156-a9e6-0c4a2cbe5181","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:05.291Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"990e2cc7-d705-4156-a9e6-0c4a2cbe5181"}
{"level":"info","ts":"2026-09-18T10:05:05.291Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"990e2cc7-d705-4156-a9e6-0c4a2cbe5181","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:05:05.295Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"33d52f65-319a-4030-88ce-5699d4705ba6","name":"delete-head-after-submit-r4xnr-workers"}
{"level":"info","ts":"2026-09-18T10:05:05.295Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"33d52f65-319a-4030-88ce-5699d4705ba6","cluster":"delete-head-after-submit-r4xnr"}
{"level":"error","ts":"2026-09-18T10:05:05.296Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"990e2cc7-d705-4156-a9e6-0c4a2cbe5181","JobId":"delete-head-after-submit-rct8k","error":"Job delete-head-after-submit-rct8k does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:05.296Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"990e2cc7-d705-4156-a9e6-0c4a2cbe5181"}
{"level":"error","ts":"2026-09-18T10:05:05.296Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"990e2cc7-d705-4156-a9e6-0c4a2cbe5181","error":"Job delete-head-after-submit-rct8k does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:05.296Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"3b7c6917-5fb2-4304-abe7-d58f98c5d7d4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:05.296Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"3b7c6917-5fb2-4304-abe7-d58f98c5d7d4","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:05.296Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"3b7c6917-5fb2-4304-abe7-d58f98c5d7d4","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:05.297Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"3b7c6917-5fb2-4304-abe7-d58f98c5d7d4"}
{"level":"info","ts":"2026-09-18T10:05:05.297Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"3b7c6917-5fb2-4304-abe7-d58f98c5d7d4","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:05.302Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"3b7c6917-5fb2-4304-abe7-d58f98c5d7d4","JobId":"delete-head-after-submit-rct8k","error":"Job delete-head-after-submit-rct8k does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:05.302Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"3b7c6917-5fb2-4304-abe7-d58f98c5d7d4"}
{"level":"error","ts":"2026-09-18T10:05:05.302Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"3b7c6917-5fb2-4304-abe7-d58f98c5d7d4","error":"Job delete-head-after-submit-rct8k does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:05.337Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"0eab1d21-36ac-4048-8040-c49b97141597","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:05.337Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"0eab1d21-36ac-4048-8040-c49b97141597","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:05.337Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"0eab1d21-36ac-4048-8040-c49b97141597","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:05.337Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"0eab1d21-36ac-4048-8040-c49b97141597"}
{"level":"info","ts":"2026-09-18T10:05:05.337Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"0eab1d21-36ac-4048-8040-c49b97141597","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:05.342Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"0eab1d21-36ac-4048-8040-c49b97141597","JobId":"delete-head-after-submit-rct8k","error":"Job delete-head-after-submit-rct8k does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:05.342Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"0eab1d21-36ac-4048-8040-c49b97141597"}
{"level":"error","ts":"2026-09-18T10:05:05.342Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"0eab1d21-36ac-4048-8040-c49b97141597","error":"Job delete-head-after-submit-rct8k does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:05.432Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"5c227624-9f56-48d7-9795-bbd35bb011c6"}
{"level":"info","ts":"2026-09-18T10:05:05.432Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"5c227624-9f56-48d7-9795-bbd35bb011c6"}
{"level":"info","ts":"2026-09-18T10:05:05.432Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"5c227624-9f56-48d7-9795-bbd35bb011c6"}
{"level":"info","ts":"2026-09-18T10:05:05.432Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"5c227624-9f56-48d7-9795-bbd35bb011c6","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:05.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"5c227624-9f56-48d7-9795-bbd35bb011c6","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:05.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"5c227624-9f56-48d7-9795-bbd35bb011c6","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:05.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"5c227624-9f56-48d7-9795-bbd35bb011c6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:05.433Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"5c227624-9f56-48d7-9795-bbd35bb011c6","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:05.433Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"5c227624-9f56-48d7-9795-bbd35bb011c6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:05.433Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"5c227624-9f56-48d7-9795-bbd35bb011c6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:05.433Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"5c227624-9f56-48d7-9795-bbd35bb011c6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:05.433Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"5c227624-9f56-48d7-9795-bbd35bb011c6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:05.433Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"5c227624-9f56-48d7-9795-bbd35bb011c6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:05:05.433Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"5c227624-9f56-48d7-9795-bbd35bb011c6","seconds":300}
{"level":"info","ts":"2026-09-18T10:05:05.493Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c30e66d9-f0e8-43ae-8684-a2553929b5a2"}
{"level":"info","ts":"2026-09-18T10:05:05.493Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c30e66d9-f0e8-43ae-8684-a2553929b5a2"}
{"level":"info","ts":"2026-09-18T10:05:05.493Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c30e66d9-f0e8-43ae-8684-a2553929b5a2"}
{"level":"info","ts":"2026-09-18T10:05:05.493Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c30e66d9-f0e8-43ae-8684-a2553929b5a2","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:05.493Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c30e66d9-f0e8-43ae-8684-a2553929b5a2","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:05.493Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c30e66d9-f0e8-43ae-8684-a2553929b5a2","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:05.493Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c30e66d9-f0e8-43ae-8684-a2553929b5a2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:05.493Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c30e66d9-f0e8-43ae-8684-a2553929b5a2","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:05.493Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c30e66d9-f0e8-43ae-8684-a2553929b5a2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:05.493Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c30e66d9-f0e8-43ae-8684-a2553929b5a2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:05.493Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c30e66d9-f0e8-43ae-8684-a2553929b5a2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:05.493Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c30e66d9-f0e8-43ae-8684-a2553929b5a2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:05.493Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c30e66d9-f0e8-43ae-8684-a2553929b5a2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:05:05.493Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c30e66d9-f0e8-43ae-8684-a2553929b5a2","seconds":300}
{"level":"info","ts":"2026-09-18T10:05:05.503Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"eb4c5fbc-eaf2-4d09-b344-ce4134504710","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:05.503Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"eb4c5fbc-eaf2-4d09-b344-ce4134504710","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:05.503Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"eb4c5fbc-eaf2-4d09-b344-ce4134504710","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:05.503Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"eb4c5fbc-eaf2-4d09-b344-ce4134504710"}
{"level":"info","ts":"2026-09-18T10:05:05.503Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"eb4c5fbc-eaf2-4d09-b344-ce4134504710","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:05:05.766Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"88dd4ad9-acca-4d0b-aa45-2225480dfd37"}
{"level":"info","ts":"2026-09-18T10:05:05.766Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"88dd4ad9-acca-4d0b-aa45-2225480dfd37"}
{"level":"info","ts":"2026-09-18T10:05:05.766Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"88dd4ad9-acca-4d0b-aa45-2225480dfd37"}
{"level":"info","ts":"2026-09-18T10:05:05.766Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"88dd4ad9-acca-4d0b-aa45-2225480dfd37","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:05.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"88dd4ad9-acca-4d0b-aa45-2225480dfd37","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:05.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"88dd4ad9-acca-4d0b-aa45-2225480dfd37","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:05.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"88dd4ad9-acca-4d0b-aa45-2225480dfd37","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:05.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"88dd4ad9-acca-4d0b-aa45-2225480dfd37","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:05.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"88dd4ad9-acca-4d0b-aa45-2225480dfd37","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:05.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"88dd4ad9-acca-4d0b-aa45-2225480dfd37","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:05.766Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"88dd4ad9-acca-4d0b-aa45-2225480dfd37","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:05.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"88dd4ad9-acca-4d0b-aa45-2225480dfd37","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:05.766Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"88dd4ad9-acca-4d0b-aa45-2225480dfd37","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:05:05.766Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"88dd4ad9-acca-4d0b-aa45-2225480dfd37","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:05:05.766Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"88dd4ad9-acca-4d0b-aa45-2225480dfd37","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:05:05.766Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"88dd4ad9-acca-4d0b-aa45-2225480dfd37","seconds":300}
{"level":"error","ts":"2026-09-18T10:05:06.492Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"eb4c5fbc-eaf2-4d09-b344-ce4134504710","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": EOF","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:06.492Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"eb4c5fbc-eaf2-4d09-b344-ce4134504710"}
{"level":"error","ts":"2026-09-18T10:05:06.492Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"eb4c5fbc-eaf2-4d09-b344-ce4134504710","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": EOF","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:06.807Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d99470e7-bd22-450c-973d-70649ddbb005"}
{"level":"info","ts":"2026-09-18T10:05:06.807Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d99470e7-bd22-450c-973d-70649ddbb005"}
{"level":"info","ts":"2026-09-18T10:05:06.807Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d99470e7-bd22-450c-973d-70649ddbb005"}
{"level":"info","ts":"2026-09-18T10:05:06.807Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d99470e7-bd22-450c-973d-70649ddbb005","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:06.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d99470e7-bd22-450c-973d-70649ddbb005","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:06.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d99470e7-bd22-450c-973d-70649ddbb005","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:06.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d99470e7-bd22-450c-973d-70649ddbb005","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:06.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d99470e7-bd22-450c-973d-70649ddbb005","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:06.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d99470e7-bd22-450c-973d-70649ddbb005","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:06.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d99470e7-bd22-450c-973d-70649ddbb005","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:06.807Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d99470e7-bd22-450c-973d-70649ddbb005","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:06.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d99470e7-bd22-450c-973d-70649ddbb005","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:06.807Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d99470e7-bd22-450c-973d-70649ddbb005","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:03:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:05:06.807Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d99470e7-bd22-450c-973d-70649ddbb005","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:03:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:05:06.807Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"d99470e7-bd22-450c-973d-70649ddbb005","name":"long-running-5qfl8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:03:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.42","serviceIP":"10.133.0.42","podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:37Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:05:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.42","serviceIP":"10.133.0.42","podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:37Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:05:06.813Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"af672de2-1b8c-4e69-822e-332afe9977e8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:06.813Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"af672de2-1b8c-4e69-822e-332afe9977e8","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:06.813Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"af672de2-1b8c-4e69-822e-332afe9977e8","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:06.813Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"af672de2-1b8c-4e69-822e-332afe9977e8"}
{"level":"info","ts":"2026-09-18T10:05:06.813Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"af672de2-1b8c-4e69-822e-332afe9977e8","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:06.815Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"af672de2-1b8c-4e69-822e-332afe9977e8","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.48:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:06.815Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"af672de2-1b8c-4e69-822e-332afe9977e8"}
{"level":"error","ts":"2026-09-18T10:05:06.815Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"af672de2-1b8c-4e69-822e-332afe9977e8","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.48:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:06.831Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"4181630a-8734-4018-8718-253079830dc5","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:05:06.831Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"4181630a-8734-4018-8718-253079830dc5","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:05:06.831Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"4181630a-8734-4018-8718-253079830dc5","pod":"long-running-5qfl8-small-group-worker-tbqtj","ip":"10.132.0.47"}
{"level":"info","ts":"2026-09-18T10:05:06.831Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"4181630a-8734-4018-8718-253079830dc5","rayCluster":"long-running-5qfl8","podIPs":["10.133.0.42","10.132.0.47"]}
{"level":"info","ts":"2026-09-18T10:05:06.831Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"aebc2f1d-afcc-47de-b27e-75238ec3e448","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:05:06.831Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"b4886aec-7bf4-465a-a6f4-2c74c8576a1e","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:06.831Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"b4886aec-7bf4-465a-a6f4-2c74c8576a1e","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-09-18 10:03:20 +0000 UTC","Now":"2026-09-18T10:05:06.831Z","ShutdownTime":"2026-09-18T10:13:20.000Z"}
{"level":"info","ts":"2026-09-18T10:05:06.831Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"b4886aec-7bf4-465a-a6f4-2c74c8576a1e","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-09-18T10:13:20.000Z","requeueAfter":495.168250051}
{"level":"info","ts":"2026-09-18T10:05:06.862Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"aebc2f1d-afcc-47de-b27e-75238ec3e448","name":"long-running-5qfl8-head"}
{"level":"info","ts":"2026-09-18T10:05:06.883Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"aebc2f1d-afcc-47de-b27e-75238ec3e448","name":"long-running-5qfl8-workers"}
{"level":"info","ts":"2026-09-18T10:05:06.883Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"aebc2f1d-afcc-47de-b27e-75238ec3e448","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:05:07.121Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"72fb2d60-de35-46e9-9b35-16e38f4d349f"}
{"level":"info","ts":"2026-09-18T10:05:07.121Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"72fb2d60-de35-46e9-9b35-16e38f4d349f"}
{"level":"info","ts":"2026-09-18T10:05:07.121Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"72fb2d60-de35-46e9-9b35-16e38f4d349f"}
{"level":"info","ts":"2026-09-18T10:05:07.121Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"72fb2d60-de35-46e9-9b35-16e38f4d349f","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:07.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"72fb2d60-de35-46e9-9b35-16e38f4d349f","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:07.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"72fb2d60-de35-46e9-9b35-16e38f4d349f","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-kw54w. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:07.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"72fb2d60-de35-46e9-9b35-16e38f4d349f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:07.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"72fb2d60-de35-46e9-9b35-16e38f4d349f","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:07.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"72fb2d60-de35-46e9-9b35-16e38f4d349f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:07.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"72fb2d60-de35-46e9-9b35-16e38f4d349f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:07.122Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"72fb2d60-de35-46e9-9b35-16e38f4d349f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:07.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"72fb2d60-de35-46e9-9b35-16e38f4d349f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:07.122Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"72fb2d60-de35-46e9-9b35-16e38f4d349f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:05:07.122Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"72fb2d60-de35-46e9-9b35-16e38f4d349f","seconds":300}
{"level":"info","ts":"2026-09-18T10:05:07.456Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2d685132-15aa-4ca7-9859-a2b829326070","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:07.456Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2d685132-15aa-4ca7-9859-a2b829326070","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:07.456Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2d685132-15aa-4ca7-9859-a2b829326070","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:07.456Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2d685132-15aa-4ca7-9859-a2b829326070"}
{"level":"info","ts":"2026-09-18T10:05:07.456Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2d685132-15aa-4ca7-9859-a2b829326070","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:07.459Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2d685132-15aa-4ca7-9859-a2b829326070","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.48:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:07.459Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2d685132-15aa-4ca7-9859-a2b829326070"}
{"level":"error","ts":"2026-09-18T10:05:07.459Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2d685132-15aa-4ca7-9859-a2b829326070","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.48:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:07.782Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a28a9f55-d1b1-42a0-9d2e-346d6c51dc4a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:07.783Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a28a9f55-d1b1-42a0-9d2e-346d6c51dc4a","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:07.783Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a28a9f55-d1b1-42a0-9d2e-346d6c51dc4a","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:07.783Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a28a9f55-d1b1-42a0-9d2e-346d6c51dc4a"}
{"level":"info","ts":"2026-09-18T10:05:07.783Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a28a9f55-d1b1-42a0-9d2e-346d6c51dc4a","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:07.785Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a28a9f55-d1b1-42a0-9d2e-346d6c51dc4a","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.48:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:07.785Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a28a9f55-d1b1-42a0-9d2e-346d6c51dc4a"}
{"level":"error","ts":"2026-09-18T10:05:07.785Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a28a9f55-d1b1-42a0-9d2e-346d6c51dc4a","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.48:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:08.624Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"159009f1-845e-447f-8e4a-80af12f618a9"}
{"level":"info","ts":"2026-09-18T10:05:08.624Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"159009f1-845e-447f-8e4a-80af12f618a9"}
{"level":"info","ts":"2026-09-18T10:05:08.624Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"159009f1-845e-447f-8e4a-80af12f618a9"}
{"level":"info","ts":"2026-09-18T10:05:08.624Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"159009f1-845e-447f-8e4a-80af12f618a9","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:08.624Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"159009f1-845e-447f-8e4a-80af12f618a9","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-kw54w","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-09-18 10:03:33 +0000 UTC,FinishedAt:2026-09-18 10:05:08 +0000 UTC,ContainerID:cri-o://867b7eb01a03cf8851130ce8601eeb12528797f743521faa057b0a26d13a28b4,}"}
{"level":"info","ts":"2026-09-18T10:05:08.624Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"159009f1-845e-447f-8e4a-80af12f618a9","head Pod":"delete-head-after-submit-r4xnr-head-kw54w","shouldDelete":true,"reason":"The head Pod delete-head-after-submit-r4xnr-head-kw54w status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-09-18T10:05:08.636Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"159009f1-845e-447f-8e4a-80af12f618a9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod delete-head-after-submit-r4xnr-head-kw54w status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:08.636Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"159009f1-845e-447f-8e4a-80af12f618a9","error":"The head Pod delete-head-after-submit-r4xnr-head-kw54w status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-09-18T10:05:08.636Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"159009f1-845e-447f-8e4a-80af12f618a9","name":"delete-head-after-submit-r4xnr","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:05:05Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:05:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:05:08Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:05:08.661Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"159009f1-845e-447f-8e4a-80af12f618a9"}
{"level":"error","ts":"2026-09-18T10:05:08.661Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"159009f1-845e-447f-8e4a-80af12f618a9","error":"The head Pod delete-head-after-submit-r4xnr-head-kw54w status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:08.661Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d5e1798e-91c8-417e-ab85-ef39d0899246","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:08.661Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d5e1798e-91c8-417e-ab85-ef39d0899246","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:08.661Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d5e1798e-91c8-417e-ab85-ef39d0899246","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:08.661Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"273873c7-3ec4-4f4b-a5dd-6e41de6170f1","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:08.661Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d5e1798e-91c8-417e-ab85-ef39d0899246"}
{"level":"info","ts":"2026-09-18T10:05:08.661Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"273873c7-3ec4-4f4b-a5dd-6e41de6170f1","pod":"delete-head-after-submit-r4xnr-head-kw54w","ip":"10.132.0.48"}
{"level":"info","ts":"2026-09-18T10:05:08.661Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"273873c7-3ec4-4f4b-a5dd-6e41de6170f1","pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","ip":"10.133.0.43"}
{"level":"info","ts":"2026-09-18T10:05:08.661Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d5e1798e-91c8-417e-ab85-ef39d0899246","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:05:08.661Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"273873c7-3ec4-4f4b-a5dd-6e41de6170f1","rayCluster":"delete-head-after-submit-r4xnr","podIPs":["10.132.0.48","10.133.0.43"]}
{"level":"info","ts":"2026-09-18T10:05:08.661Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"c9273b9e-6f09-4976-969d-d6eb47edbf16","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:08.666Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"28b84f2c-d7cf-4a16-a07d-82bd9921b38b"}
{"level":"info","ts":"2026-09-18T10:05:08.666Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"28b84f2c-d7cf-4a16-a07d-82bd9921b38b"}
{"level":"info","ts":"2026-09-18T10:05:08.666Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"28b84f2c-d7cf-4a16-a07d-82bd9921b38b"}
{"level":"info","ts":"2026-09-18T10:05:08.666Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"28b84f2c-d7cf-4a16-a07d-82bd9921b38b","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:08.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"28b84f2c-d7cf-4a16-a07d-82bd9921b38b","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-09-18T10:05:08.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"28b84f2c-d7cf-4a16-a07d-82bd9921b38b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:08.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"28b84f2c-d7cf-4a16-a07d-82bd9921b38b","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:08.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"28b84f2c-d7cf-4a16-a07d-82bd9921b38b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:08.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"28b84f2c-d7cf-4a16-a07d-82bd9921b38b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:08.667Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"28b84f2c-d7cf-4a16-a07d-82bd9921b38b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:08.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"28b84f2c-d7cf-4a16-a07d-82bd9921b38b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:08.667Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"28b84f2c-d7cf-4a16-a07d-82bd9921b38b","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-09-18T10:05:08.667Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"28b84f2c-d7cf-4a16-a07d-82bd9921b38b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:05:08.667Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"28b84f2c-d7cf-4a16-a07d-82bd9921b38b","seconds":300}
{"level":"info","ts":"2026-09-18T10:05:08.694Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"c9273b9e-6f09-4976-969d-d6eb47edbf16","name":"delete-head-after-submit-r4xnr-head"}
{"level":"info","ts":"2026-09-18T10:05:08.717Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"c9273b9e-6f09-4976-969d-d6eb47edbf16","name":"delete-head-after-submit-r4xnr-workers"}
{"level":"info","ts":"2026-09-18T10:05:08.717Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"c9273b9e-6f09-4976-969d-d6eb47edbf16","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:08.831Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"7628325f-93e3-40f9-a150-5c7cf9d3356a"}
{"level":"info","ts":"2026-09-18T10:05:08.831Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"7628325f-93e3-40f9-a150-5c7cf9d3356a"}
{"level":"info","ts":"2026-09-18T10:05:08.831Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"7628325f-93e3-40f9-a150-5c7cf9d3356a"}
{"level":"info","ts":"2026-09-18T10:05:08.831Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"7628325f-93e3-40f9-a150-5c7cf9d3356a","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:08.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"7628325f-93e3-40f9-a150-5c7cf9d3356a","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:08.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"7628325f-93e3-40f9-a150-5c7cf9d3356a","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:08.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"7628325f-93e3-40f9-a150-5c7cf9d3356a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:08.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"7628325f-93e3-40f9-a150-5c7cf9d3356a","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:08.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"7628325f-93e3-40f9-a150-5c7cf9d3356a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:08.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"7628325f-93e3-40f9-a150-5c7cf9d3356a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:08.832Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"7628325f-93e3-40f9-a150-5c7cf9d3356a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:08.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"7628325f-93e3-40f9-a150-5c7cf9d3356a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:08.832Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"7628325f-93e3-40f9-a150-5c7cf9d3356a","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:03:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:05:08.832Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"7628325f-93e3-40f9-a150-5c7cf9d3356a","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:03:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:05:08.832Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"7628325f-93e3-40f9-a150-5c7cf9d3356a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:05:08.832Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"7628325f-93e3-40f9-a150-5c7cf9d3356a","seconds":300}
{"level":"info","ts":"2026-09-18T10:05:08.834Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c6292baf-492a-4256-b054-21f2d309f749"}
{"level":"info","ts":"2026-09-18T10:05:08.834Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c6292baf-492a-4256-b054-21f2d309f749"}
{"level":"info","ts":"2026-09-18T10:05:08.834Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c6292baf-492a-4256-b054-21f2d309f749"}
{"level":"info","ts":"2026-09-18T10:05:08.834Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c6292baf-492a-4256-b054-21f2d309f749","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:08.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c6292baf-492a-4256-b054-21f2d309f749","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-09-18T10:05:08.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c6292baf-492a-4256-b054-21f2d309f749","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:08.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c6292baf-492a-4256-b054-21f2d309f749","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:08.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c6292baf-492a-4256-b054-21f2d309f749","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:08.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c6292baf-492a-4256-b054-21f2d309f749","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:08.834Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c6292baf-492a-4256-b054-21f2d309f749","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:08.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c6292baf-492a-4256-b054-21f2d309f749","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:08.834Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c6292baf-492a-4256-b054-21f2d309f749","podName":"delete-head-after-submit-r4xnr-head-kw54w","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-09-18T10:05:08.834Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c6292baf-492a-4256-b054-21f2d309f749","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:05:08.834Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"c6292baf-492a-4256-b054-21f2d309f749","seconds":300}
{"level":"info","ts":"2026-09-18T10:05:08.846Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564"}
{"level":"info","ts":"2026-09-18T10:05:08.846Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564"}
{"level":"info","ts":"2026-09-18T10:05:08.846Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564"}
{"level":"info","ts":"2026-09-18T10:05:08.846Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:08.846Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564"}
{"level":"info","ts":"2026-09-18T10:05:08.865Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:05:08.866Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564"}
{"level":"info","ts":"2026-09-18T10:05:08.866Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","mode":"IntegratedOAuth","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:08.866Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","cluster":"delete-head-after-submit-r4xnr","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:05:08.866Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","cluster":"delete-head-after-submit-r4xnr","authType":"IntegratedOAuth","serviceAccount":"delete-head-after-submit-r4xnr-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T10:05:08.866Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"delete-head-after-submit-r4xnr","ray.io/group":"headgroup","ray.io/identifier":"delete-head-after-submit-r4xnr-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:05:08.866Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T10:05:08.866Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:05:08.866Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:05:08.866Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","enabled":true}
{"level":"info","ts":"2026-09-18T10:05:08.891Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","name":"delete-head-after-submit-r4xnr-head-bvh54"}
{"level":"info","ts":"2026-09-18T10:05:08.891Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:08.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:08.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:08.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:08.892Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:08.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:08.892Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","podName":"delete-head-after-submit-r4xnr-head-bvh54","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:05:08.892Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"14e72b45-82a9-4b3f-b5d9-95aa87b5d564","name":"delete-head-after-submit-r4xnr","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:05:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:05:08Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:05:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-bvh54","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:05:08Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:05:08.918Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41"}
{"level":"info","ts":"2026-09-18T10:05:08.918Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41"}
{"level":"info","ts":"2026-09-18T10:05:08.918Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41"}
{"level":"info","ts":"2026-09-18T10:05:08.918Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:08.918Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"2fa689a6-ce19-4591-9d75-8e1a0b59253f","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:08.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-bvh54","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:08.918Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"2fa689a6-ce19-4591-9d75-8e1a0b59253f","pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","ip":"10.133.0.43"}
{"level":"info","ts":"2026-09-18T10:05:08.918Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"aecf439a-230f-4fe5-a4d2-31dd1a8047e7","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:08.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41","head Pod":"delete-head-after-submit-r4xnr-head-bvh54","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-bvh54. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:08.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:08.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:08.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:08.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:08.918Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:08.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:08.918Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41","podName":"delete-head-after-submit-r4xnr-head-bvh54","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:05:08.919Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41","name":"delete-head-after-submit-r4xnr","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:05:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:05:08Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:05:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-bvh54","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:05:08Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:05:08.941Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41","name":"delete-head-after-submit-r4xnr","error":"Operation cannot be fulfilled on rayclusters.ray.io \"delete-head-after-submit-r4xnr\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr"}}
{"level":"info","ts":"2026-09-18T10:05:08.941Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41"}
{"level":"error","ts":"2026-09-18T10:05:08.941Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"4179861e-1632-4224-aaa6-cc3031732c41","error":"Operation cannot be fulfilled on rayclusters.ray.io \"delete-head-after-submit-r4xnr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:08.946Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"608ba6a5-b87f-4451-817a-494b4178b94d"}
{"level":"info","ts":"2026-09-18T10:05:08.946Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"608ba6a5-b87f-4451-817a-494b4178b94d"}
{"level":"info","ts":"2026-09-18T10:05:08.946Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"608ba6a5-b87f-4451-817a-494b4178b94d"}
{"level":"info","ts":"2026-09-18T10:05:08.946Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"608ba6a5-b87f-4451-817a-494b4178b94d","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:08.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"608ba6a5-b87f-4451-817a-494b4178b94d","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-bvh54","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:08.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"608ba6a5-b87f-4451-817a-494b4178b94d","head Pod":"delete-head-after-submit-r4xnr-head-bvh54","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-bvh54. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:08.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"608ba6a5-b87f-4451-817a-494b4178b94d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:08.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"608ba6a5-b87f-4451-817a-494b4178b94d","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:08.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"608ba6a5-b87f-4451-817a-494b4178b94d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:08.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"608ba6a5-b87f-4451-817a-494b4178b94d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:08.946Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"608ba6a5-b87f-4451-817a-494b4178b94d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:08.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"608ba6a5-b87f-4451-817a-494b4178b94d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:08.946Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"608ba6a5-b87f-4451-817a-494b4178b94d","podName":"delete-head-after-submit-r4xnr-head-bvh54","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:05:08.946Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"608ba6a5-b87f-4451-817a-494b4178b94d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:05:08.946Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"608ba6a5-b87f-4451-817a-494b4178b94d","seconds":300}
{"level":"info","ts":"2026-09-18T10:05:08.977Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"aecf439a-230f-4fe5-a4d2-31dd1a8047e7","name":"delete-head-after-submit-r4xnr-head"}
{"level":"info","ts":"2026-09-18T10:05:08.980Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b4511e2-40ae-4f99-be9f-9029b2dd7a82"}
{"level":"info","ts":"2026-09-18T10:05:08.980Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b4511e2-40ae-4f99-be9f-9029b2dd7a82"}
{"level":"info","ts":"2026-09-18T10:05:08.980Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b4511e2-40ae-4f99-be9f-9029b2dd7a82"}
{"level":"info","ts":"2026-09-18T10:05:08.980Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b4511e2-40ae-4f99-be9f-9029b2dd7a82","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:08.980Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b4511e2-40ae-4f99-be9f-9029b2dd7a82","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-bvh54","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:08.980Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b4511e2-40ae-4f99-be9f-9029b2dd7a82","head Pod":"delete-head-after-submit-r4xnr-head-bvh54","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-bvh54. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:08.980Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b4511e2-40ae-4f99-be9f-9029b2dd7a82","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:08.980Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b4511e2-40ae-4f99-be9f-9029b2dd7a82","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:08.980Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b4511e2-40ae-4f99-be9f-9029b2dd7a82","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:08.980Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b4511e2-40ae-4f99-be9f-9029b2dd7a82","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:08.980Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b4511e2-40ae-4f99-be9f-9029b2dd7a82","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:08.980Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b4511e2-40ae-4f99-be9f-9029b2dd7a82","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:08.980Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b4511e2-40ae-4f99-be9f-9029b2dd7a82","podName":"delete-head-after-submit-r4xnr-head-bvh54","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:05:08.980Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b4511e2-40ae-4f99-be9f-9029b2dd7a82","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:05:08.980Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"2b4511e2-40ae-4f99-be9f-9029b2dd7a82","seconds":300}
{"level":"info","ts":"2026-09-18T10:05:08.987Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"2fa689a6-ce19-4591-9d75-8e1a0b59253f","certificate":"ray-head-cert-delete-head-after-submit-r4xnr","dnsNames":["delete-head-after-submit-r4xnr-head-svc","localhost","delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc","delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.43","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:05:09.007Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"43010257-5696-47d0-823b-901a1fe5ac32"}
{"level":"info","ts":"2026-09-18T10:05:09.007Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"43010257-5696-47d0-823b-901a1fe5ac32"}
{"level":"info","ts":"2026-09-18T10:05:09.007Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"43010257-5696-47d0-823b-901a1fe5ac32"}
{"level":"info","ts":"2026-09-18T10:05:09.007Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"43010257-5696-47d0-823b-901a1fe5ac32","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:09.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"43010257-5696-47d0-823b-901a1fe5ac32","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-bvh54","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:09.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"43010257-5696-47d0-823b-901a1fe5ac32","head Pod":"delete-head-after-submit-r4xnr-head-bvh54","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-bvh54. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:09.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"43010257-5696-47d0-823b-901a1fe5ac32","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:09.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"43010257-5696-47d0-823b-901a1fe5ac32","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:09.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"43010257-5696-47d0-823b-901a1fe5ac32","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:09.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"43010257-5696-47d0-823b-901a1fe5ac32","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:09.007Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"43010257-5696-47d0-823b-901a1fe5ac32","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:09.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"43010257-5696-47d0-823b-901a1fe5ac32","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:09.007Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"43010257-5696-47d0-823b-901a1fe5ac32","podName":"delete-head-after-submit-r4xnr-head-bvh54","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:05:09.007Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"43010257-5696-47d0-823b-901a1fe5ac32","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:05:09.007Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"43010257-5696-47d0-823b-901a1fe5ac32","seconds":300}
{"level":"info","ts":"2026-09-18T10:05:09.045Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"2fa689a6-ce19-4591-9d75-8e1a0b59253f","certificate":"ray-worker-cert-delete-head-after-submit-r4xnr","dnsNames":["delete-head-after-submit-r4xnr-worker-svc","localhost","delete-head-after-submit-r4xnr-small-group","delete-head-after-submit-r4xnr-worker-svc.test-ns-ltx56.svc","delete-head-after-submit-r4xnr-worker-svc.test-ns-ltx56.svc.cluster.local","*.delete-head-after-submit-r4xnr-worker-svc.test-ns-ltx56.svc","*.delete-head-after-submit-r4xnr-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.43","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:05:09.045Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"2fa689a6-ce19-4591-9d75-8e1a0b59253f","rayCluster":"delete-head-after-submit-r4xnr","podIPs":["10.133.0.43"]}
{"level":"info","ts":"2026-09-18T10:05:09.046Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"aecf439a-230f-4fe5-a4d2-31dd1a8047e7","name":"delete-head-after-submit-r4xnr-workers"}
{"level":"info","ts":"2026-09-18T10:05:09.046Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"aecf439a-230f-4fe5-a4d2-31dd1a8047e7","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:09.054Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f23b4696-6e67-4302-8f1a-37a0b6bddec8"}
{"level":"info","ts":"2026-09-18T10:05:09.054Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f23b4696-6e67-4302-8f1a-37a0b6bddec8"}
{"level":"info","ts":"2026-09-18T10:05:09.054Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f23b4696-6e67-4302-8f1a-37a0b6bddec8"}
{"level":"info","ts":"2026-09-18T10:05:09.054Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f23b4696-6e67-4302-8f1a-37a0b6bddec8","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:09.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f23b4696-6e67-4302-8f1a-37a0b6bddec8","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-bvh54","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:09.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f23b4696-6e67-4302-8f1a-37a0b6bddec8","head Pod":"delete-head-after-submit-r4xnr-head-bvh54","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-bvh54. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:09.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f23b4696-6e67-4302-8f1a-37a0b6bddec8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:09.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f23b4696-6e67-4302-8f1a-37a0b6bddec8","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:09.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f23b4696-6e67-4302-8f1a-37a0b6bddec8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:09.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f23b4696-6e67-4302-8f1a-37a0b6bddec8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:09.054Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f23b4696-6e67-4302-8f1a-37a0b6bddec8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:09.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f23b4696-6e67-4302-8f1a-37a0b6bddec8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:09.054Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f23b4696-6e67-4302-8f1a-37a0b6bddec8","podName":"delete-head-after-submit-r4xnr-head-bvh54","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:05:09.054Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"f23b4696-6e67-4302-8f1a-37a0b6bddec8","name":"delete-head-after-submit-r4xnr","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:05:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-bvh54","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:05:08Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:05:09Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-bvh54","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:05:08Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:05:09.078Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"698472ba-4e31-4891-aed9-fa9d833127ff","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:09.078Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"822d9d4d-dfd5-4bb2-b067-fb221d9b34d4","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:09.078Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"698472ba-4e31-4891-aed9-fa9d833127ff","pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","ip":"10.133.0.43"}
{"level":"info","ts":"2026-09-18T10:05:09.078Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"698472ba-4e31-4891-aed9-fa9d833127ff","rayCluster":"delete-head-after-submit-r4xnr","podIPs":["10.133.0.43"]}
{"level":"info","ts":"2026-09-18T10:05:09.122Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"822d9d4d-dfd5-4bb2-b067-fb221d9b34d4","name":"delete-head-after-submit-r4xnr-head"}
{"level":"info","ts":"2026-09-18T10:05:09.177Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"822d9d4d-dfd5-4bb2-b067-fb221d9b34d4","name":"delete-head-after-submit-r4xnr-workers"}
{"level":"info","ts":"2026-09-18T10:05:09.177Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"822d9d4d-dfd5-4bb2-b067-fb221d9b34d4","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:09.622Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6dfe4a6a-360e-4007-bc53-587a837f9396"}
{"level":"info","ts":"2026-09-18T10:05:09.622Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6dfe4a6a-360e-4007-bc53-587a837f9396"}
{"level":"info","ts":"2026-09-18T10:05:09.622Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6dfe4a6a-360e-4007-bc53-587a837f9396"}
{"level":"info","ts":"2026-09-18T10:05:09.622Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6dfe4a6a-360e-4007-bc53-587a837f9396","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:09.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6dfe4a6a-360e-4007-bc53-587a837f9396","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-bvh54","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:09.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6dfe4a6a-360e-4007-bc53-587a837f9396","head Pod":"delete-head-after-submit-r4xnr-head-bvh54","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-bvh54. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:09.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6dfe4a6a-360e-4007-bc53-587a837f9396","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:09.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6dfe4a6a-360e-4007-bc53-587a837f9396","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:09.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6dfe4a6a-360e-4007-bc53-587a837f9396","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:09.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6dfe4a6a-360e-4007-bc53-587a837f9396","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:09.623Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6dfe4a6a-360e-4007-bc53-587a837f9396","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:09.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6dfe4a6a-360e-4007-bc53-587a837f9396","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:09.623Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6dfe4a6a-360e-4007-bc53-587a837f9396","podName":"delete-head-after-submit-r4xnr-head-bvh54","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:05:09.623Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6dfe4a6a-360e-4007-bc53-587a837f9396","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:05:09.623Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"6dfe4a6a-360e-4007-bc53-587a837f9396","seconds":300}
{"level":"info","ts":"2026-09-18T10:05:09.814Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"fb0bd5a1-58ce-4a41-aede-5cfb5595cbc4"}
{"level":"info","ts":"2026-09-18T10:05:09.814Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"fb0bd5a1-58ce-4a41-aede-5cfb5595cbc4"}
{"level":"info","ts":"2026-09-18T10:05:09.814Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"fb0bd5a1-58ce-4a41-aede-5cfb5595cbc4"}
{"level":"info","ts":"2026-09-18T10:05:09.814Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"fb0bd5a1-58ce-4a41-aede-5cfb5595cbc4","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:09.814Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"fb0bd5a1-58ce-4a41-aede-5cfb5595cbc4","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-bvh54","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:09.814Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"fb0bd5a1-58ce-4a41-aede-5cfb5595cbc4","head Pod":"delete-head-after-submit-r4xnr-head-bvh54","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-bvh54. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:09.814Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"fb0bd5a1-58ce-4a41-aede-5cfb5595cbc4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:09.814Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"fb0bd5a1-58ce-4a41-aede-5cfb5595cbc4","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:09.814Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"fb0bd5a1-58ce-4a41-aede-5cfb5595cbc4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:09.814Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"fb0bd5a1-58ce-4a41-aede-5cfb5595cbc4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:09.814Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"fb0bd5a1-58ce-4a41-aede-5cfb5595cbc4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:09.814Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"fb0bd5a1-58ce-4a41-aede-5cfb5595cbc4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:09.814Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"fb0bd5a1-58ce-4a41-aede-5cfb5595cbc4","podName":"delete-head-after-submit-r4xnr-head-bvh54","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:05:09.814Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"fb0bd5a1-58ce-4a41-aede-5cfb5595cbc4","name":"delete-head-after-submit-r4xnr","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:05:09Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-r4xnr-head-bvh54","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:05:08Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:05:09Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.50","serviceIP":"10.132.0.50","podName":"delete-head-after-submit-r4xnr-head-bvh54","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:05:08Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:05:09.842Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"b357bd7d-7b4e-4c2c-8794-e34d361efaa4","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:09.842Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"b357bd7d-7b4e-4c2c-8794-e34d361efaa4","pod":"delete-head-after-submit-r4xnr-head-bvh54","ip":"10.132.0.50"}
{"level":"info","ts":"2026-09-18T10:05:09.842Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"b357bd7d-7b4e-4c2c-8794-e34d361efaa4","pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","ip":"10.133.0.43"}
{"level":"info","ts":"2026-09-18T10:05:09.842Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"3ec66d00-de2b-45d7-bff1-91a05465f2da","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:09.862Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"b357bd7d-7b4e-4c2c-8794-e34d361efaa4","certificate":"ray-head-cert-delete-head-after-submit-r4xnr","dnsNames":["delete-head-after-submit-r4xnr-head-svc","localhost","delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc","delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.50","10.133.0.43","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:05:09.871Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"3ec66d00-de2b-45d7-bff1-91a05465f2da","name":"delete-head-after-submit-r4xnr-head"}
{"level":"info","ts":"2026-09-18T10:05:09.899Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"b357bd7d-7b4e-4c2c-8794-e34d361efaa4","certificate":"ray-worker-cert-delete-head-after-submit-r4xnr","dnsNames":["delete-head-after-submit-r4xnr-worker-svc","localhost","delete-head-after-submit-r4xnr-small-group","delete-head-after-submit-r4xnr-worker-svc.test-ns-ltx56.svc","delete-head-after-submit-r4xnr-worker-svc.test-ns-ltx56.svc.cluster.local","*.delete-head-after-submit-r4xnr-worker-svc.test-ns-ltx56.svc","*.delete-head-after-submit-r4xnr-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.50","10.133.0.43","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:05:09.899Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"b357bd7d-7b4e-4c2c-8794-e34d361efaa4","rayCluster":"delete-head-after-submit-r4xnr","podIPs":["10.132.0.50","10.133.0.43"]}
{"level":"info","ts":"2026-09-18T10:05:09.901Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"3ec66d00-de2b-45d7-bff1-91a05465f2da","name":"delete-head-after-submit-r4xnr-workers"}
{"level":"info","ts":"2026-09-18T10:05:09.901Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"3ec66d00-de2b-45d7-bff1-91a05465f2da","cluster":"delete-head-after-submit-r4xnr"}
{"level":"error","ts":"2026-09-18T10:05:10.661Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d5e1798e-91c8-417e-ab85-ef39d0899246","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:10.662Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d5e1798e-91c8-417e-ab85-ef39d0899246"}
{"level":"error","ts":"2026-09-18T10:05:10.662Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d5e1798e-91c8-417e-ab85-ef39d0899246","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:10.662Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef641e30-9fab-4a24-bcb2-5c38d0e26c34","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:10.662Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef641e30-9fab-4a24-bcb2-5c38d0e26c34","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:10.662Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef641e30-9fab-4a24-bcb2-5c38d0e26c34","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:10.662Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef641e30-9fab-4a24-bcb2-5c38d0e26c34"}
{"level":"info","ts":"2026-09-18T10:05:10.662Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef641e30-9fab-4a24-bcb2-5c38d0e26c34","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:10.666Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef641e30-9fab-4a24-bcb2-5c38d0e26c34","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:10.666Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef641e30-9fab-4a24-bcb2-5c38d0e26c34"}
{"level":"error","ts":"2026-09-18T10:05:10.666Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"ef641e30-9fab-4a24-bcb2-5c38d0e26c34","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:11.076Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"280af7a4-f9b5-4bc4-b8f6-31893e821a5e"}
{"level":"info","ts":"2026-09-18T10:05:11.076Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"280af7a4-f9b5-4bc4-b8f6-31893e821a5e"}
{"level":"info","ts":"2026-09-18T10:05:11.076Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"280af7a4-f9b5-4bc4-b8f6-31893e821a5e"}
{"level":"info","ts":"2026-09-18T10:05:11.076Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"280af7a4-f9b5-4bc4-b8f6-31893e821a5e","1 head service found":"delete-head-after-submit-r4xnr-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:11.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"280af7a4-f9b5-4bc4-b8f6-31893e821a5e","Found 1 head Pod":"delete-head-after-submit-r4xnr-head-bvh54","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:11.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"280af7a4-f9b5-4bc4-b8f6-31893e821a5e","head Pod":"delete-head-after-submit-r4xnr-head-bvh54","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-r4xnr-head-bvh54. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:11.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"280af7a4-f9b5-4bc4-b8f6-31893e821a5e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:11.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"280af7a4-f9b5-4bc4-b8f6-31893e821a5e","worker Pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-r4xnr-small-group-worker-ww4rm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:11.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"280af7a4-f9b5-4bc4-b8f6-31893e821a5e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:11.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"280af7a4-f9b5-4bc4-b8f6-31893e821a5e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:11.076Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"280af7a4-f9b5-4bc4-b8f6-31893e821a5e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:11.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"280af7a4-f9b5-4bc4-b8f6-31893e821a5e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:11.076Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"280af7a4-f9b5-4bc4-b8f6-31893e821a5e","podName":"delete-head-after-submit-r4xnr-head-bvh54","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:05:11.076Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"280af7a4-f9b5-4bc4-b8f6-31893e821a5e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:05:11.076Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"280af7a4-f9b5-4bc4-b8f6-31893e821a5e","seconds":300}
{"level":"info","ts":"2026-09-18T10:05:15.782Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d4d23068-1ada-49e9-b81b-ddde5f2f8f4a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:15.782Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d4d23068-1ada-49e9-b81b-ddde5f2f8f4a","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:15.782Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d4d23068-1ada-49e9-b81b-ddde5f2f8f4a","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:15.782Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d4d23068-1ada-49e9-b81b-ddde5f2f8f4a"}
{"level":"info","ts":"2026-09-18T10:05:15.782Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d4d23068-1ada-49e9-b81b-ddde5f2f8f4a","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:15.785Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d4d23068-1ada-49e9-b81b-ddde5f2f8f4a","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:15.786Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d4d23068-1ada-49e9-b81b-ddde5f2f8f4a"}
{"level":"error","ts":"2026-09-18T10:05:15.786Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d4d23068-1ada-49e9-b81b-ddde5f2f8f4a","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:16.293Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"55903154-5c42-4edf-bd19-8c2c9316c165","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:16.293Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"55903154-5c42-4edf-bd19-8c2c9316c165","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:16.293Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"55903154-5c42-4edf-bd19-8c2c9316c165","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:16.293Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"55903154-5c42-4edf-bd19-8c2c9316c165"}
{"level":"info","ts":"2026-09-18T10:05:16.293Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"55903154-5c42-4edf-bd19-8c2c9316c165","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:16.296Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"55903154-5c42-4edf-bd19-8c2c9316c165","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:16.296Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"55903154-5c42-4edf-bd19-8c2c9316c165"}
{"level":"error","ts":"2026-09-18T10:05:16.296Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"55903154-5c42-4edf-bd19-8c2c9316c165","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:16.962Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"67f6fd89-372a-4732-aa37-4af3b5ed3620","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:16.962Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"67f6fd89-372a-4732-aa37-4af3b5ed3620","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:16.962Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"67f6fd89-372a-4732-aa37-4af3b5ed3620","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:16.962Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"67f6fd89-372a-4732-aa37-4af3b5ed3620"}
{"level":"info","ts":"2026-09-18T10:05:16.962Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"67f6fd89-372a-4732-aa37-4af3b5ed3620","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:16.964Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"67f6fd89-372a-4732-aa37-4af3b5ed3620","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:16.964Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"67f6fd89-372a-4732-aa37-4af3b5ed3620"}
{"level":"error","ts":"2026-09-18T10:05:16.964Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"67f6fd89-372a-4732-aa37-4af3b5ed3620","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:17.170Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"bdfbcd12-9538-440c-8120-32a5be8bf3a8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:17.170Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"bdfbcd12-9538-440c-8120-32a5be8bf3a8","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:17.170Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"bdfbcd12-9538-440c-8120-32a5be8bf3a8","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:17.170Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"bdfbcd12-9538-440c-8120-32a5be8bf3a8"}
{"level":"info","ts":"2026-09-18T10:05:17.170Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"bdfbcd12-9538-440c-8120-32a5be8bf3a8","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:17.172Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"bdfbcd12-9538-440c-8120-32a5be8bf3a8","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:17.173Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"bdfbcd12-9538-440c-8120-32a5be8bf3a8"}
{"level":"error","ts":"2026-09-18T10:05:17.173Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"bdfbcd12-9538-440c-8120-32a5be8bf3a8","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:17.805Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"6e015923-b3f5-40a3-bf8c-aa7eb9895d6c"}
{"level":"info","ts":"2026-09-18T10:05:17.805Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"6e015923-b3f5-40a3-bf8c-aa7eb9895d6c"}
{"level":"info","ts":"2026-09-18T10:05:17.805Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"6e015923-b3f5-40a3-bf8c-aa7eb9895d6c"}
{"level":"info","ts":"2026-09-18T10:05:17.805Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"6e015923-b3f5-40a3-bf8c-aa7eb9895d6c","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:17.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"6e015923-b3f5-40a3-bf8c-aa7eb9895d6c","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:17.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"6e015923-b3f5-40a3-bf8c-aa7eb9895d6c","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:17.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"6e015923-b3f5-40a3-bf8c-aa7eb9895d6c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:17.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"6e015923-b3f5-40a3-bf8c-aa7eb9895d6c","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:17.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"6e015923-b3f5-40a3-bf8c-aa7eb9895d6c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:17.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"6e015923-b3f5-40a3-bf8c-aa7eb9895d6c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:17.805Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"6e015923-b3f5-40a3-bf8c-aa7eb9895d6c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:17.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"6e015923-b3f5-40a3-bf8c-aa7eb9895d6c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:17.805Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"6e015923-b3f5-40a3-bf8c-aa7eb9895d6c","name":"long-running-5qfl8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:05:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.42","serviceIP":"10.133.0.42","podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:37Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:05:17Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.42","serviceIP":"10.133.0.42","podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:37Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:05:17.832Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"d077a26b-c9b2-45de-b128-40526bc1cc39","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:05:17.832Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"bffdceb8-fcc3-4e2c-bcb7-7a0e75c19933","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:05:17.833Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"bffdceb8-fcc3-4e2c-bcb7-7a0e75c19933","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:05:17.833Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"bffdceb8-fcc3-4e2c-bcb7-7a0e75c19933","pod":"long-running-5qfl8-small-group-worker-tbqtj","ip":"10.132.0.47"}
{"level":"info","ts":"2026-09-18T10:05:17.833Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"bffdceb8-fcc3-4e2c-bcb7-7a0e75c19933","rayCluster":"long-running-5qfl8","podIPs":["10.133.0.42","10.132.0.47"]}
{"level":"info","ts":"2026-09-18T10:05:17.833Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"7f3cd778-4ca1-4f4e-9d10-44924feb5f4d","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:17.833Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"7f3cd778-4ca1-4f4e-9d10-44924feb5f4d","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-09-18 10:03:20 +0000 UTC","Now":"2026-09-18T10:05:17.833Z","ShutdownTime":"2026-09-18T10:13:20.000Z"}
{"level":"info","ts":"2026-09-18T10:05:17.833Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"7f3cd778-4ca1-4f4e-9d10-44924feb5f4d","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-09-18T10:13:20.000Z","requeueAfter":484.166680056}
{"level":"info","ts":"2026-09-18T10:05:17.855Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"d077a26b-c9b2-45de-b128-40526bc1cc39","name":"long-running-5qfl8-head"}
{"level":"info","ts":"2026-09-18T10:05:17.874Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"d077a26b-c9b2-45de-b128-40526bc1cc39","name":"long-running-5qfl8-workers"}
{"level":"info","ts":"2026-09-18T10:05:17.874Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"d077a26b-c9b2-45de-b128-40526bc1cc39","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:05:19.833Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0a028134-5519-4bd7-817f-386f638f77cb"}
{"level":"info","ts":"2026-09-18T10:05:19.833Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0a028134-5519-4bd7-817f-386f638f77cb"}
{"level":"info","ts":"2026-09-18T10:05:19.833Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0a028134-5519-4bd7-817f-386f638f77cb"}
{"level":"info","ts":"2026-09-18T10:05:19.833Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0a028134-5519-4bd7-817f-386f638f77cb","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:05:19.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0a028134-5519-4bd7-817f-386f638f77cb","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:05:19.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0a028134-5519-4bd7-817f-386f638f77cb","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:19.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0a028134-5519-4bd7-817f-386f638f77cb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:05:19.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0a028134-5519-4bd7-817f-386f638f77cb","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:05:19.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0a028134-5519-4bd7-817f-386f638f77cb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:19.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0a028134-5519-4bd7-817f-386f638f77cb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:05:19.833Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0a028134-5519-4bd7-817f-386f638f77cb","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:05:19.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0a028134-5519-4bd7-817f-386f638f77cb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:05:19.833Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0a028134-5519-4bd7-817f-386f638f77cb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:05:19.833Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0a028134-5519-4bd7-817f-386f638f77cb","seconds":300}
{"level":"info","ts":"2026-09-18T10:05:24.034Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"56321c6f-08c3-4e55-9915-7f4e9edddb70","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:24.034Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"56321c6f-08c3-4e55-9915-7f4e9edddb70","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:24.034Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"56321c6f-08c3-4e55-9915-7f4e9edddb70","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:24.035Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"56321c6f-08c3-4e55-9915-7f4e9edddb70"}
{"level":"info","ts":"2026-09-18T10:05:24.035Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"56321c6f-08c3-4e55-9915-7f4e9edddb70","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:24.038Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"56321c6f-08c3-4e55-9915-7f4e9edddb70","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:24.038Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"56321c6f-08c3-4e55-9915-7f4e9edddb70"}
{"level":"error","ts":"2026-09-18T10:05:24.038Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"56321c6f-08c3-4e55-9915-7f4e9edddb70","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:26.191Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"fd618248-2f7e-472c-abe3-4b62af15bb15","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:26.191Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"fd618248-2f7e-472c-abe3-4b62af15bb15","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:26.191Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"fd618248-2f7e-472c-abe3-4b62af15bb15","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:26.191Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"fd618248-2f7e-472c-abe3-4b62af15bb15"}
{"level":"info","ts":"2026-09-18T10:05:26.191Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"fd618248-2f7e-472c-abe3-4b62af15bb15","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:26.193Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"fd618248-2f7e-472c-abe3-4b62af15bb15","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:26.193Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"fd618248-2f7e-472c-abe3-4b62af15bb15"}
{"level":"error","ts":"2026-09-18T10:05:26.193Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"fd618248-2f7e-472c-abe3-4b62af15bb15","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:33.633Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"30c1211f-58a7-40d2-afaf-caef1880ace9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:33.633Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"30c1211f-58a7-40d2-afaf-caef1880ace9","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:33.633Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"30c1211f-58a7-40d2-afaf-caef1880ace9","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:33.633Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"30c1211f-58a7-40d2-afaf-caef1880ace9"}
{"level":"info","ts":"2026-09-18T10:05:33.633Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"30c1211f-58a7-40d2-afaf-caef1880ace9","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:33.636Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"30c1211f-58a7-40d2-afaf-caef1880ace9","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:33.636Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"30c1211f-58a7-40d2-afaf-caef1880ace9"}
{"level":"error","ts":"2026-09-18T10:05:33.636Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"30c1211f-58a7-40d2-afaf-caef1880ace9","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:33.670Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"67be004d-6bab-42ef-8f35-757052c0e01b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:33.670Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"67be004d-6bab-42ef-8f35-757052c0e01b","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:33.670Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"67be004d-6bab-42ef-8f35-757052c0e01b","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:33.670Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"67be004d-6bab-42ef-8f35-757052c0e01b"}
{"level":"info","ts":"2026-09-18T10:05:33.670Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"67be004d-6bab-42ef-8f35-757052c0e01b","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:33.673Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"67be004d-6bab-42ef-8f35-757052c0e01b","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:33.673Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"67be004d-6bab-42ef-8f35-757052c0e01b"}
{"level":"error","ts":"2026-09-18T10:05:33.673Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"67be004d-6bab-42ef-8f35-757052c0e01b","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:36.267Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"7ac2d1a6-29bc-40de-875e-ac7288e01f27","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:36.267Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"7ac2d1a6-29bc-40de-875e-ac7288e01f27","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:36.267Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"7ac2d1a6-29bc-40de-875e-ac7288e01f27","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:36.267Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"7ac2d1a6-29bc-40de-875e-ac7288e01f27"}
{"level":"info","ts":"2026-09-18T10:05:36.267Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"7ac2d1a6-29bc-40de-875e-ac7288e01f27","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:36.269Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"7ac2d1a6-29bc-40de-875e-ac7288e01f27","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:36.269Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"7ac2d1a6-29bc-40de-875e-ac7288e01f27"}
{"level":"error","ts":"2026-09-18T10:05:36.269Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"7ac2d1a6-29bc-40de-875e-ac7288e01f27","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:45.579Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"7a9063f9-cf4e-432a-8985-2e2269f4108d","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:05:45.579Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"7a9063f9-cf4e-432a-8985-2e2269f4108d","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:05:45.579Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"7a9063f9-cf4e-432a-8985-2e2269f4108d","pod":"long-running-5qfl8-small-group-worker-tbqtj","ip":"10.132.0.47"}
{"level":"info","ts":"2026-09-18T10:05:45.579Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"7a9063f9-cf4e-432a-8985-2e2269f4108d","rayCluster":"long-running-5qfl8","podIPs":["10.133.0.42","10.132.0.47"]}
{"level":"info","ts":"2026-09-18T10:05:51.506Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"dae1307e-6a55-4389-a158-bae1d9cf8e52","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:51.506Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"dae1307e-6a55-4389-a158-bae1d9cf8e52","pod":"delete-head-after-submit-r4xnr-head-bvh54","ip":"10.132.0.50"}
{"level":"info","ts":"2026-09-18T10:05:51.506Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"dae1307e-6a55-4389-a158-bae1d9cf8e52","pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","ip":"10.133.0.43"}
{"level":"info","ts":"2026-09-18T10:05:51.506Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"dae1307e-6a55-4389-a158-bae1d9cf8e52","rayCluster":"delete-head-after-submit-r4xnr","podIPs":["10.132.0.50","10.133.0.43"]}
{"level":"info","ts":"2026-09-18T10:05:52.035Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a5df0903-ded8-4089-a7c0-b725516a2f64","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:52.035Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a5df0903-ded8-4089-a7c0-b725516a2f64","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:52.035Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a5df0903-ded8-4089-a7c0-b725516a2f64","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:52.035Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a5df0903-ded8-4089-a7c0-b725516a2f64"}
{"level":"info","ts":"2026-09-18T10:05:52.035Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a5df0903-ded8-4089-a7c0-b725516a2f64","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:52.040Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a5df0903-ded8-4089-a7c0-b725516a2f64","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:52.040Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a5df0903-ded8-4089-a7c0-b725516a2f64"}
{"level":"error","ts":"2026-09-18T10:05:52.040Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"a5df0903-ded8-4089-a7c0-b725516a2f64","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:53.029Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"79f6fffb-29fd-48a9-959c-e78bc1478767","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:05:53.029Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"79f6fffb-29fd-48a9-959c-e78bc1478767","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:05:53.029Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"79f6fffb-29fd-48a9-959c-e78bc1478767","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:05:53.029Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"79f6fffb-29fd-48a9-959c-e78bc1478767"}
{"level":"info","ts":"2026-09-18T10:05:53.029Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"79f6fffb-29fd-48a9-959c-e78bc1478767","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:05:53.032Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"79f6fffb-29fd-48a9-959c-e78bc1478767","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:05:53.032Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"79f6fffb-29fd-48a9-959c-e78bc1478767"}
{"level":"error","ts":"2026-09-18T10:05:53.032Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"79f6fffb-29fd-48a9-959c-e78bc1478767","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:02.049Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"330f8caf-4461-4781-be20-526b2237806f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:02.050Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"330f8caf-4461-4781-be20-526b2237806f","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:02.050Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"330f8caf-4461-4781-be20-526b2237806f","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:02.050Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"330f8caf-4461-4781-be20-526b2237806f"}
{"level":"info","ts":"2026-09-18T10:06:02.050Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"330f8caf-4461-4781-be20-526b2237806f","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:06:02.053Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"330f8caf-4461-4781-be20-526b2237806f","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:02.053Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"330f8caf-4461-4781-be20-526b2237806f"}
{"level":"error","ts":"2026-09-18T10:06:02.053Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"330f8caf-4461-4781-be20-526b2237806f","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:03.061Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d2b02e17-a278-4882-a376-da37d87b6661","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:03.061Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d2b02e17-a278-4882-a376-da37d87b6661","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:03.061Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d2b02e17-a278-4882-a376-da37d87b6661","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:03.061Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d2b02e17-a278-4882-a376-da37d87b6661"}
{"level":"info","ts":"2026-09-18T10:06:03.061Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d2b02e17-a278-4882-a376-da37d87b6661","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:06:03.064Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d2b02e17-a278-4882-a376-da37d87b6661","JobId":"delete-head-after-submit-rct8k","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:03.064Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d2b02e17-a278-4882-a376-da37d87b6661"}
{"level":"error","ts":"2026-09-18T10:06:03.064Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"d2b02e17-a278-4882-a376-da37d87b6661","error":"Get \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:03.100Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"f1ed1955-f378-49ee-8abe-17d507a2c787","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:03.100Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"f1ed1955-f378-49ee-8abe-17d507a2c787","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:03.100Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"f1ed1955-f378-49ee-8abe-17d507a2c787","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:03.100Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"f1ed1955-f378-49ee-8abe-17d507a2c787"}
{"level":"info","ts":"2026-09-18T10:06:03.100Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"f1ed1955-f378-49ee-8abe-17d507a2c787","rayCluster":"delete-head-after-submit-r4xnr","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:06:03.100Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has failed. Attempting to transition the status to `Failed`.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"f1ed1955-f378-49ee-8abe-17d507a2c787","Submitter K8s Job":"delete-head-after-submit","Reason":"BackoffLimitExceeded","Message":"Job has reached the specified backoff limit"}
{"level":"info","ts":"2026-09-18T10:06:03.100Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"f1ed1955-f378-49ee-8abe-17d507a2c787","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","dashboardURL":"delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-rct8k","rayClusterName":"delete-head-after-submit-r4xnr","dashboardURL":"delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Failed","reason":"SubmissionFailed","message":"Job submission has failed. Reason: BackoffLimitExceeded. Message: Job has reached the specified backoff limit","startTime":"2026-09-18T10:03:21Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:04:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.48","serviceIP":"10.132.0.48","podName":"delete-head-after-submit-r4xnr-head-kw54w","serviceName":"delete-head-after-submit-r4xnr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:03.100Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"f1ed1955-f378-49ee-8abe-17d507a2c787","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-09-18T10:06:03.126Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4b5831ea-84d3-41e5-86bf-be146128b8c9","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:03.126Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4b5831ea-84d3-41e5-86bf-be146128b8c9","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:06:03 +0000 UTC","Now":"2026-09-18T10:06:03.126Z","ShutdownTime":"2026-09-18T10:06:03.000Z"}
{"level":"info","ts":"2026-09-18T10:06:03.149Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"d9eaa76d-81a7-4ff7-9797-9e44341265e4"}
{"level":"info","ts":"2026-09-18T10:06:03.149Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"reconcileID":"7939f118-9dde-4fac-86f6-e651f5444fc8"}
{"level":"info","ts":"2026-09-18T10:06:03.149Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"165cfbf3-4778-41bf-a6cf-ea24b000f02b","namespacedName":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:03.149Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"165cfbf3-4778-41bf-a6cf-ea24b000f02b","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:03.149Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"165cfbf3-4778-41bf-a6cf-ea24b000f02b","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:03.149Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"5c19ca62-00a9-4ed9-9a5d-935e91a25e57","rayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:03.150Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"165cfbf3-4778-41bf-a6cf-ea24b000f02b","namespace":"test-ns-ltx56","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-09-18T10:06:03.150Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"165cfbf3-4778-41bf-a6cf-ea24b000f02b","name":"kuberay-gateway-access","namespace":"test-ns-ltx56","remainingClusters":1}
{"level":"info","ts":"2026-09-18T10:06:03.150Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"5c19ca62-00a9-4ed9-9a5d-935e91a25e57","pod":"delete-head-after-submit-r4xnr-head-bvh54","ip":"10.132.0.50"}
{"level":"info","ts":"2026-09-18T10:06:03.150Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"5c19ca62-00a9-4ed9-9a5d-935e91a25e57","pod":"delete-head-after-submit-r4xnr-small-group-worker-ww4rm","ip":"10.133.0.43"}
{"level":"info","ts":"2026-09-18T10:06:03.150Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"5c19ca62-00a9-4ed9-9a5d-935e91a25e57","rayCluster":"delete-head-after-submit-r4xnr","podIPs":["10.132.0.50","10.133.0.43"]}
{"level":"info","ts":"2026-09-18T10:06:03.151Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4b5831ea-84d3-41e5-86bf-be146128b8c9","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:03.151Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4b5831ea-84d3-41e5-86bf-be146128b8c9","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T10:06:03.151Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"4b5831ea-84d3-41e5-86bf-be146128b8c9","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:03.151Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"21198e9b-9c69-413f-bbde-ea90dbee05bd","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:03.151Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"21198e9b-9c69-413f-bbde-ea90dbee05bd","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:06:03 +0000 UTC","Now":"2026-09-18T10:06:03.151Z","ShutdownTime":"2026-09-18T10:06:03.000Z"}
{"level":"info","ts":"2026-09-18T10:06:03.151Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"21198e9b-9c69-413f-bbde-ea90dbee05bd","RayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:03.151Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"21198e9b-9c69-413f-bbde-ea90dbee05bd","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T10:06:03.151Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"21198e9b-9c69-413f-bbde-ea90dbee05bd","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:03.160Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"165cfbf3-4778-41bf-a6cf-ea24b000f02b","name":"test-ns-ltx56-delete-head-after-submit-r4xnr","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T10:06:03.172Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"165cfbf3-4778-41bf-a6cf-ea24b000f02b","configMap":"kube-rbac-proxy-config-delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:03.180Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"165cfbf3-4778-41bf-a6cf-ea24b000f02b","serviceAccount":"delete-head-after-submit-r4xnr-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:06:03.221Z","logger":"raycluster-resource","msg":"default","name":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:03.221Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:06:03.239Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"429f191c-605d-4449-95c0-4ffc64329781","namespace":"test-ns-ltx56","clusterName":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:03.239Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"dafb1afe-e12c-4844-8b77-794df95c2d27","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:03.239Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"dafb1afe-e12c-4844-8b77-794df95c2d27","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:06:03 +0000 UTC","Now":"2026-09-18T10:06:03.239Z","ShutdownTime":"2026-09-18T10:06:03.000Z"}
{"level":"info","ts":"2026-09-18T10:06:03.239Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"dafb1afe-e12c-4844-8b77-794df95c2d27","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:03.239Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"dafb1afe-e12c-4844-8b77-794df95c2d27","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T10:06:03.239Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"dafb1afe-e12c-4844-8b77-794df95c2d27","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:03.239Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"0cf22aa6-21d7-4df4-b2fe-da66108ddbbf"}
{"level":"info","ts":"2026-09-18T10:06:03.243Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"165cfbf3-4778-41bf-a6cf-ea24b000f02b","cluster":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:03.243Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"c733e236-3d49-43cb-8d63-6dc2bf8a7d56","namespacedName":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:03.243Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"c733e236-3d49-43cb-8d63-6dc2bf8a7d56"}
{"level":"info","ts":"2026-09-18T10:06:03.243Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"c733e236-3d49-43cb-8d63-6dc2bf8a7d56","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:06:03.243Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"c733e236-3d49-43cb-8d63-6dc2bf8a7d56","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:06:03.264Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"27c7618b-5021-4878-b87b-4eb1c7fa76c6"}
{"level":"info","ts":"2026-09-18T10:06:03.275Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"f444ee8b-88a5-4c75-a402-f26fddadb12d"}
{"level":"info","ts":"2026-09-18T10:06:03.307Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"60effadc-4a83-4697-8486-17cf1ee171de","namespacedName":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:03.307Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"60effadc-4a83-4697-8486-17cf1ee171de"}
{"level":"info","ts":"2026-09-18T10:06:03.307Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"60effadc-4a83-4697-8486-17cf1ee171de","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:06:03.307Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"60effadc-4a83-4697-8486-17cf1ee171de","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:06:03.353Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"429f191c-605d-4449-95c0-4ffc64329781","namespace":"test-ns-ltx56","clusterName":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:04.048Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"473aca4c-4c8c-4e52-b59a-2765bd4ccd83","DeletionTimestamp":"2026-09-18 10:06:04 +0000 UTC"}
{"level":"error","ts":"2026-09-18T10:06:04.049Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"473aca4c-4c8c-4e52-b59a-2765bd4ccd83","error":"RayCluster.ray.io \"delete-head-after-submit-r4xnr\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:121\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:06:04.051Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"473aca4c-4c8c-4e52-b59a-2765bd4ccd83","error":"Post \"http://delete-head-after-submit-r4xnr-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs/delete-head-after-submit-rct8k/stop\": dial tcp 10.132.0.50:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.051Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"473aca4c-4c8c-4e52-b59a-2765bd4ccd83","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T10:06:04.101Z","logger":"controllers.RayJob","msg":"Skipping RayJob managed by a custom controller","RayJob":{"name":"managed-externally","namespace":"test-ns-ltx56"},"reconcileID":"776cf256-066e-4650-94fd-beeb7c24de4c","managed-by":"kueue.x-k8s.io/multikueue"}
{"level":"info","ts":"2026-09-18T10:06:04.102Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"f6a62b94-9644-4fb8-b6ea-f5a7f7daf62d"}
{"level":"info","ts":"2026-09-18T10:06:04.119Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"56eb938b-0afb-4335-ab11-9825de8d7b6a"}
{"level":"info","ts":"2026-09-18T10:06:04.121Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"96b4b789-20ee-4bbf-ae35-ccb0e3088c6c"}
{"level":"info","ts":"2026-09-18T10:06:04.166Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"managed-externally","namespace":"test-ns-ltx56"},"reconcileID":"ded8c696-4f4c-47de-a842-28e60b56ea59"}
{"level":"info","ts":"2026-09-18T10:06:04.189Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"96803e60-a5c8-4204-be0d-7cad43475baa","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:04.189Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"96803e60-a5c8-4204-be0d-7cad43475baa","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T10:06:04.209Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"96803e60-a5c8-4204-be0d-7cad43475baa"}
{"level":"info","ts":"2026-09-18T10:06:04.209Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"96803e60-a5c8-4204-be0d-7cad43475baa","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T10:06:04.209Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"96803e60-a5c8-4204-be0d-7cad43475baa","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:06:04.209Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"96803e60-a5c8-4204-be0d-7cad43475baa","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:06:04.226Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b4246ba7-ab79-4923-81ed-df95cfb19d77","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:04.226Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b4246ba7-ab79-4923-81ed-df95cfb19d77","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.226Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b4246ba7-ab79-4923-81ed-df95cfb19d77","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:04.226Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b4246ba7-ab79-4923-81ed-df95cfb19d77","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:04.235Z","logger":"raycluster-resource","msg":"default","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.235Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:06:04.261Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"1d8da428-f9e3-465f-ba25-e92cb4723632","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.261Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"081177fd-0df0-4efc-97bb-43ddf1f16c7d"}
{"level":"info","ts":"2026-09-18T10:06:04.261Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"081177fd-0df0-4efc-97bb-43ddf1f16c7d"}
{"level":"info","ts":"2026-09-18T10:06:04.261Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"081177fd-0df0-4efc-97bb-43ddf1f16c7d"}
{"level":"info","ts":"2026-09-18T10:06:04.262Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"35726ee0-3d6f-4dd2-b29e-1a4703e77cb1","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.262Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"35726ee0-3d6f-4dd2-b29e-1a4703e77cb1","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.262Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"a55c2b16-6f2d-40e0-881d-64e149bf6f06","namespacedName":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:04.262Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"a55c2b16-6f2d-40e0-881d-64e149bf6f06","mode":"IntegratedOAuth","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.262Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"a55c2b16-6f2d-40e0-881d-64e149bf6f06","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.262Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"a55c2b16-6f2d-40e0-881d-64e149bf6f06","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.263Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b4246ba7-ab79-4923-81ed-df95cfb19d77","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:04.263Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b4246ba7-ab79-4923-81ed-df95cfb19d77"}
{"level":"info","ts":"2026-09-18T10:06:04.263Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b4246ba7-ab79-4923-81ed-df95cfb19d77","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:04.263Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b4246ba7-ab79-4923-81ed-df95cfb19d77","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:06:04.263Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b8e9701c-378b-4a72-b2c0-b7092eae3438","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:04.263Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b8e9701c-378b-4a72-b2c0-b7092eae3438","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.263Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b8e9701c-378b-4a72-b2c0-b7092eae3438","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:04.263Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b8e9701c-378b-4a72-b2c0-b7092eae3438"}
{"level":"info","ts":"2026-09-18T10:06:04.263Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b8e9701c-378b-4a72-b2c0-b7092eae3438","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:04.263Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b8e9701c-378b-4a72-b2c0-b7092eae3438","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:06:04.268Z","logger":"raycluster-resource","msg":"default","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.268Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:06:04.273Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"081177fd-0df0-4efc-97bb-43ddf1f16c7d","name":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:04.273Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"081177fd-0df0-4efc-97bb-43ddf1f16c7d","error":"head certificate secret ray-head-secret-submitter-timeout-lgrnm not found"}
{"level":"error","ts":"2026-09-18T10:06:04.273Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"081177fd-0df0-4efc-97bb-43ddf1f16c7d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.273Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"081177fd-0df0-4efc-97bb-43ddf1f16c7d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found"}
{"level":"info","ts":"2026-09-18T10:06:04.273Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"081177fd-0df0-4efc-97bb-43ddf1f16c7d","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.273Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"081177fd-0df0-4efc-97bb-43ddf1f16c7d","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.273Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"081177fd-0df0-4efc-97bb-43ddf1f16c7d","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.273Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"081177fd-0df0-4efc-97bb-43ddf1f16c7d","name":"submitter-timeout-lgrnm","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:06:04.285Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"35726ee0-3d6f-4dd2-b29e-1a4703e77cb1","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.285Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"35726ee0-3d6f-4dd2-b29e-1a4703e77cb1","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.289Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"50d34775-0979-4bbe-ad48-c60df4d33a24","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:04.289Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"50d34775-0979-4bbe-ad48-c60df4d33a24","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.289Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"50d34775-0979-4bbe-ad48-c60df4d33a24","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:04.289Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"50d34775-0979-4bbe-ad48-c60df4d33a24"}
{"level":"info","ts":"2026-09-18T10:06:04.289Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"50d34775-0979-4bbe-ad48-c60df4d33a24","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:04.289Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"50d34775-0979-4bbe-ad48-c60df4d33a24","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:06:04.297Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"081177fd-0df0-4efc-97bb-43ddf1f16c7d","name":"submitter-timeout-lgrnm","error":"Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-lgrnm\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm"}}
{"level":"info","ts":"2026-09-18T10:06:04.297Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"081177fd-0df0-4efc-97bb-43ddf1f16c7d"}
{"level":"error","ts":"2026-09-18T10:06:04.297Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"081177fd-0df0-4efc-97bb-43ddf1f16c7d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.297Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"1d8da428-f9e3-465f-ba25-e92cb4723632","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.297Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"666335f1-1ca0-440e-82ab-a238eac96cf2","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.297Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e39c10c5-adf4-430a-827b-940eb2b1dd1a"}
{"level":"info","ts":"2026-09-18T10:06:04.297Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e39c10c5-adf4-430a-827b-940eb2b1dd1a"}
{"level":"info","ts":"2026-09-18T10:06:04.297Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e39c10c5-adf4-430a-827b-940eb2b1dd1a"}
{"level":"info","ts":"2026-09-18T10:06:04.297Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e39c10c5-adf4-430a-827b-940eb2b1dd1a","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:04.297Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e39c10c5-adf4-430a-827b-940eb2b1dd1a","error":"head certificate secret ray-head-secret-submitter-timeout-lgrnm not found"}
{"level":"error","ts":"2026-09-18T10:06:04.297Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e39c10c5-adf4-430a-827b-940eb2b1dd1a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.297Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e39c10c5-adf4-430a-827b-940eb2b1dd1a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found"}
{"level":"info","ts":"2026-09-18T10:06:04.297Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e39c10c5-adf4-430a-827b-940eb2b1dd1a","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.297Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e39c10c5-adf4-430a-827b-940eb2b1dd1a","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.297Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e39c10c5-adf4-430a-827b-940eb2b1dd1a","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.297Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e39c10c5-adf4-430a-827b-940eb2b1dd1a","name":"submitter-timeout-lgrnm","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:06:04.308Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"a55c2b16-6f2d-40e0-881d-64e149bf6f06","name":"submitter-timeout-lgrnm-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:06:04.323Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"666335f1-1ca0-440e-82ab-a238eac96cf2","name":"submitter-timeout-lgrnm-head"}
{"level":"info","ts":"2026-09-18T10:06:04.330Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"a55c2b16-6f2d-40e0-881d-64e149bf6f06","name":"test-ns-ltx56-submitter-timeout-lgrnm","operation":"created"}
{"level":"info","ts":"2026-09-18T10:06:04.331Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e39c10c5-adf4-430a-827b-940eb2b1dd1a"}
{"level":"error","ts":"2026-09-18T10:06:04.331Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e39c10c5-adf4-430a-827b-940eb2b1dd1a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.331Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"52c56b1b-3e8a-4a63-bcad-8df265fc03dd"}
{"level":"info","ts":"2026-09-18T10:06:04.331Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"52c56b1b-3e8a-4a63-bcad-8df265fc03dd"}
{"level":"info","ts":"2026-09-18T10:06:04.331Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"52c56b1b-3e8a-4a63-bcad-8df265fc03dd"}
{"level":"info","ts":"2026-09-18T10:06:04.331Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"52c56b1b-3e8a-4a63-bcad-8df265fc03dd","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:04.331Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"52c56b1b-3e8a-4a63-bcad-8df265fc03dd","error":"head certificate secret ray-head-secret-submitter-timeout-lgrnm not found"}
{"level":"error","ts":"2026-09-18T10:06:04.331Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"52c56b1b-3e8a-4a63-bcad-8df265fc03dd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.331Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"52c56b1b-3e8a-4a63-bcad-8df265fc03dd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found"}
{"level":"info","ts":"2026-09-18T10:06:04.331Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"52c56b1b-3e8a-4a63-bcad-8df265fc03dd","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.331Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"52c56b1b-3e8a-4a63-bcad-8df265fc03dd","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.331Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"52c56b1b-3e8a-4a63-bcad-8df265fc03dd","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.331Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"52c56b1b-3e8a-4a63-bcad-8df265fc03dd","name":"submitter-timeout-lgrnm","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:06:04.337Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"35726ee0-3d6f-4dd2-b29e-1a4703e77cb1","rayCluster":"submitter-timeout-lgrnm","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T10:06:04.337Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"35726ee0-3d6f-4dd2-b29e-1a4703e77cb1","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.356Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"a55c2b16-6f2d-40e0-881d-64e149bf6f06","name":"kube-rbac-proxy-config-submitter-timeout-lgrnm","operation":"created"}
{"level":"info","ts":"2026-09-18T10:06:04.359Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"35726ee0-3d6f-4dd2-b29e-1a4703e77cb1","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.331Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c7f91ded-66f5-4d69-8bbe-65126be21383","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:04.365Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c7f91ded-66f5-4d69-8bbe-65126be21383","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.366Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c7f91ded-66f5-4d69-8bbe-65126be21383","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:04.366Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c7f91ded-66f5-4d69-8bbe-65126be21383"}
{"level":"info","ts":"2026-09-18T10:06:04.366Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c7f91ded-66f5-4d69-8bbe-65126be21383","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:04.366Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c7f91ded-66f5-4d69-8bbe-65126be21383","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:04.366Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c7f91ded-66f5-4d69-8bbe-65126be21383","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:06:04.368Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"52c56b1b-3e8a-4a63-bcad-8df265fc03dd","name":"submitter-timeout-lgrnm","error":"Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-lgrnm\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm"}}
{"level":"info","ts":"2026-09-18T10:06:04.368Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"52c56b1b-3e8a-4a63-bcad-8df265fc03dd"}
{"level":"error","ts":"2026-09-18T10:06:04.368Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"52c56b1b-3e8a-4a63-bcad-8df265fc03dd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.368Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"472f8a36-0790-4e6b-9cd9-4100a0938fe4"}
{"level":"info","ts":"2026-09-18T10:06:04.368Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"472f8a36-0790-4e6b-9cd9-4100a0938fe4"}
{"level":"info","ts":"2026-09-18T10:06:04.368Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"472f8a36-0790-4e6b-9cd9-4100a0938fe4"}
{"level":"info","ts":"2026-09-18T10:06:04.368Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"472f8a36-0790-4e6b-9cd9-4100a0938fe4","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:04.368Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"472f8a36-0790-4e6b-9cd9-4100a0938fe4","error":"head certificate secret ray-head-secret-submitter-timeout-lgrnm not found"}
{"level":"error","ts":"2026-09-18T10:06:04.368Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"472f8a36-0790-4e6b-9cd9-4100a0938fe4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.368Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"472f8a36-0790-4e6b-9cd9-4100a0938fe4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found"}
{"level":"info","ts":"2026-09-18T10:06:04.368Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"472f8a36-0790-4e6b-9cd9-4100a0938fe4","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.368Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"472f8a36-0790-4e6b-9cd9-4100a0938fe4","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.368Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"472f8a36-0790-4e6b-9cd9-4100a0938fe4","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.368Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"472f8a36-0790-4e6b-9cd9-4100a0938fe4"}
{"level":"error","ts":"2026-09-18T10:06:04.368Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"472f8a36-0790-4e6b-9cd9-4100a0938fe4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:06:04.382Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"a55c2b16-6f2d-40e0-881d-64e149bf6f06","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-lgrnm\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.382Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"a55c2b16-6f2d-40e0-881d-64e149bf6f06"}
{"level":"error","ts":"2026-09-18T10:06:04.382Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"a55c2b16-6f2d-40e0-881d-64e149bf6f06","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-lgrnm\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.382Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"c52e76ce-4f3d-4f6c-9ed0-f4d812a8e09f","namespacedName":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:04.382Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"c52e76ce-4f3d-4f6c-9ed0-f4d812a8e09f","mode":"IntegratedOAuth","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.382Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"c52e76ce-4f3d-4f6c-9ed0-f4d812a8e09f","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.388Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"bad5aa4e-6c9a-4e7c-a446-4017ecef845b"}
{"level":"info","ts":"2026-09-18T10:06:04.388Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"bad5aa4e-6c9a-4e7c-a446-4017ecef845b"}
{"level":"info","ts":"2026-09-18T10:06:04.388Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"bad5aa4e-6c9a-4e7c-a446-4017ecef845b"}
{"level":"info","ts":"2026-09-18T10:06:04.388Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"bad5aa4e-6c9a-4e7c-a446-4017ecef845b","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:04.388Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"bad5aa4e-6c9a-4e7c-a446-4017ecef845b","error":"head certificate secret ray-head-secret-submitter-timeout-lgrnm not found"}
{"level":"error","ts":"2026-09-18T10:06:04.388Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"bad5aa4e-6c9a-4e7c-a446-4017ecef845b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.388Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"bad5aa4e-6c9a-4e7c-a446-4017ecef845b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found"}
{"level":"info","ts":"2026-09-18T10:06:04.388Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"bad5aa4e-6c9a-4e7c-a446-4017ecef845b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.388Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"bad5aa4e-6c9a-4e7c-a446-4017ecef845b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.388Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"bad5aa4e-6c9a-4e7c-a446-4017ecef845b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.388Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"bad5aa4e-6c9a-4e7c-a446-4017ecef845b"}
{"level":"error","ts":"2026-09-18T10:06:04.388Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"bad5aa4e-6c9a-4e7c-a446-4017ecef845b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.390Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"666335f1-1ca0-440e-82ab-a238eac96cf2","name":"submitter-timeout-lgrnm-workers"}
{"level":"info","ts":"2026-09-18T10:06:04.390Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"666335f1-1ca0-440e-82ab-a238eac96cf2","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.390Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"00bfd4a6-9138-45a3-b7c3-8811123e5c04","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.395Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0f16c377-b6f1-40e6-88d9-40af6145c931","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:04.395Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0f16c377-b6f1-40e6-88d9-40af6145c931","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.395Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0f16c377-b6f1-40e6-88d9-40af6145c931","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:04.395Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0f16c377-b6f1-40e6-88d9-40af6145c931"}
{"level":"info","ts":"2026-09-18T10:06:04.395Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0f16c377-b6f1-40e6-88d9-40af6145c931","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:04.395Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0f16c377-b6f1-40e6-88d9-40af6145c931","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:04.405Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"c52e76ce-4f3d-4f6c-9ed0-f4d812a8e09f","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.405Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"74d42bde-e816-4036-9ee2-7d7a26e65a49","namespacedName":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:04.405Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"74d42bde-e816-4036-9ee2-7d7a26e65a49","mode":"IntegratedOAuth","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.405Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"74d42bde-e816-4036-9ee2-7d7a26e65a49","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.407Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"93782d7a-d718-4e8b-888b-e46d06d67a73","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:04.407Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"93782d7a-d718-4e8b-888b-e46d06d67a73","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.407Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"93782d7a-d718-4e8b-888b-e46d06d67a73","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:04.407Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"93782d7a-d718-4e8b-888b-e46d06d67a73"}
{"level":"info","ts":"2026-09-18T10:06:04.407Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"93782d7a-d718-4e8b-888b-e46d06d67a73","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:04.408Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"93782d7a-d718-4e8b-888b-e46d06d67a73","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:04.408Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"93782d7a-d718-4e8b-888b-e46d06d67a73","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:06:04.412Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"35726ee0-3d6f-4dd2-b29e-1a4703e77cb1","rayCluster":"submitter-timeout-lgrnm","podIPs":[]}
{"level":"error","ts":"2026-09-18T10:06:04.419Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"74d42bde-e816-4036-9ee2-7d7a26e65a49","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-lgrnm\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.419Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"74d42bde-e816-4036-9ee2-7d7a26e65a49"}
{"level":"error","ts":"2026-09-18T10:06:04.419Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"74d42bde-e816-4036-9ee2-7d7a26e65a49","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-lgrnm\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.466Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6344433a-7009-4ed2-aaaf-f2a574417d3b","namespacedName":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:04.466Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6344433a-7009-4ed2-aaaf-f2a574417d3b","mode":"IntegratedOAuth","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.466Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6344433a-7009-4ed2-aaaf-f2a574417d3b","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.467Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"00bfd4a6-9138-45a3-b7c3-8811123e5c04","name":"submitter-timeout-lgrnm-head"}
{"level":"info","ts":"2026-09-18T10:06:04.467Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"35726ee0-3d6f-4dd2-b29e-1a4703e77cb1","rayCluster":"submitter-timeout-lgrnm","podIPs":[]}
{"level":"error","ts":"2026-09-18T10:06:04.467Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"35726ee0-3d6f-4dd2-b29e-1a4703e77cb1","error":"Certificate.cert-manager.io \"ray-worker-cert-submitter-timeout-lgrnm\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.467Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"35726ee0-3d6f-4dd2-b29e-1a4703e77cb1"}
{"level":"error","ts":"2026-09-18T10:06:04.467Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"35726ee0-3d6f-4dd2-b29e-1a4703e77cb1","error":"Certificate.cert-manager.io \"ray-worker-cert-submitter-timeout-lgrnm\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.467Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"401dc3d9-e308-4f35-bfd8-2789bf033fad","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.468Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"2b5d9ad6-3c7c-4744-bb60-1bc7e0718010","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:04.468Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"2b5d9ad6-3c7c-4744-bb60-1bc7e0718010","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.468Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"2b5d9ad6-3c7c-4744-bb60-1bc7e0718010","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:04.469Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"2b5d9ad6-3c7c-4744-bb60-1bc7e0718010"}
{"level":"info","ts":"2026-09-18T10:06:04.469Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"2b5d9ad6-3c7c-4744-bb60-1bc7e0718010","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:04.469Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"2b5d9ad6-3c7c-4744-bb60-1bc7e0718010","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:04.567Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6344433a-7009-4ed2-aaaf-f2a574417d3b","cluster":"submitter-timeout-lgrnm"}
{"level":"error","ts":"2026-09-18T10:06:04.568Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"401dc3d9-e308-4f35-bfd8-2789bf033fad","error":"certificates.cert-manager.io \"ray-worker-cert-submitter-timeout-lgrnm\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:06:04.568Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"401dc3d9-e308-4f35-bfd8-2789bf033fad","error":"certificates.cert-manager.io \"ray-worker-cert-submitter-timeout-lgrnm\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.568Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"401dc3d9-e308-4f35-bfd8-2789bf033fad"}
{"level":"error","ts":"2026-09-18T10:06:04.568Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"401dc3d9-e308-4f35-bfd8-2789bf033fad","error":"certificates.cert-manager.io \"ray-worker-cert-submitter-timeout-lgrnm\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.568Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"46634031-7249-47b4-8a8c-ea8da17ef5f2","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.568Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"46634031-7249-47b4-8a8c-ea8da17ef5f2"}
{"level":"info","ts":"2026-09-18T10:06:04.666Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ef7cb0a3-f6c3-4188-ab61-f1f792d0e43e"}
{"level":"info","ts":"2026-09-18T10:06:04.666Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ef7cb0a3-f6c3-4188-ab61-f1f792d0e43e"}
{"level":"info","ts":"2026-09-18T10:06:04.666Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ef7cb0a3-f6c3-4188-ab61-f1f792d0e43e"}
{"level":"info","ts":"2026-09-18T10:06:04.666Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ef7cb0a3-f6c3-4188-ab61-f1f792d0e43e","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:04.666Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ef7cb0a3-f6c3-4188-ab61-f1f792d0e43e","error":"head certificate secret ray-head-secret-submitter-timeout-lgrnm not found"}
{"level":"error","ts":"2026-09-18T10:06:04.666Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ef7cb0a3-f6c3-4188-ab61-f1f792d0e43e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.666Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ef7cb0a3-f6c3-4188-ab61-f1f792d0e43e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found"}
{"level":"info","ts":"2026-09-18T10:06:04.666Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ef7cb0a3-f6c3-4188-ab61-f1f792d0e43e","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.666Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ef7cb0a3-f6c3-4188-ab61-f1f792d0e43e","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.666Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ef7cb0a3-f6c3-4188-ab61-f1f792d0e43e","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.666Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ef7cb0a3-f6c3-4188-ab61-f1f792d0e43e"}
{"level":"error","ts":"2026-09-18T10:06:04.666Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ef7cb0a3-f6c3-4188-ab61-f1f792d0e43e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.667Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"c40d7761-1597-4d4b-86ef-d40621e6b8c3","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.667Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"c40d7761-1597-4d4b-86ef-d40621e6b8c3"}
{"level":"info","ts":"2026-09-18T10:06:04.668Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"00bfd4a6-9138-45a3-b7c3-8811123e5c04","name":"submitter-timeout-lgrnm-workers"}
{"level":"info","ts":"2026-09-18T10:06:04.668Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"00bfd4a6-9138-45a3-b7c3-8811123e5c04","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.668Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"5eb74928-98e5-489e-a151-d67e33532b1a","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.772Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"5eb74928-98e5-489e-a151-d67e33532b1a","name":"submitter-timeout-lgrnm-head"}
{"level":"info","ts":"2026-09-18T10:06:04.795Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"5eb74928-98e5-489e-a151-d67e33532b1a","name":"submitter-timeout-lgrnm-workers"}
{"level":"info","ts":"2026-09-18T10:06:04.795Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"5eb74928-98e5-489e-a151-d67e33532b1a","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:04.827Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"38ae5dde-64b2-4294-80dc-ea266330cf27"}
{"level":"info","ts":"2026-09-18T10:06:04.827Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"38ae5dde-64b2-4294-80dc-ea266330cf27"}
{"level":"info","ts":"2026-09-18T10:06:04.827Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"38ae5dde-64b2-4294-80dc-ea266330cf27"}
{"level":"info","ts":"2026-09-18T10:06:04.827Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"38ae5dde-64b2-4294-80dc-ea266330cf27","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:04.827Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"38ae5dde-64b2-4294-80dc-ea266330cf27","error":"head certificate secret ray-head-secret-submitter-timeout-lgrnm not found"}
{"level":"error","ts":"2026-09-18T10:06:04.827Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"38ae5dde-64b2-4294-80dc-ea266330cf27","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:04.827Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"38ae5dde-64b2-4294-80dc-ea266330cf27","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found"}
{"level":"info","ts":"2026-09-18T10:06:04.827Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"38ae5dde-64b2-4294-80dc-ea266330cf27","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.827Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"38ae5dde-64b2-4294-80dc-ea266330cf27","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.827Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"38ae5dde-64b2-4294-80dc-ea266330cf27","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:04.827Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"38ae5dde-64b2-4294-80dc-ea266330cf27"}
{"level":"error","ts":"2026-09-18T10:06:04.827Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"38ae5dde-64b2-4294-80dc-ea266330cf27","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:05.148Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8924bc4f-bb78-4e10-9133-feca9422fed7"}
{"level":"info","ts":"2026-09-18T10:06:05.148Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8924bc4f-bb78-4e10-9133-feca9422fed7"}
{"level":"info","ts":"2026-09-18T10:06:05.148Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8924bc4f-bb78-4e10-9133-feca9422fed7"}
{"level":"info","ts":"2026-09-18T10:06:05.148Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8924bc4f-bb78-4e10-9133-feca9422fed7","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:05.148Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8924bc4f-bb78-4e10-9133-feca9422fed7","error":"head certificate secret ray-head-secret-submitter-timeout-lgrnm not found"}
{"level":"error","ts":"2026-09-18T10:06:05.148Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8924bc4f-bb78-4e10-9133-feca9422fed7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:05.148Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8924bc4f-bb78-4e10-9133-feca9422fed7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found"}
{"level":"info","ts":"2026-09-18T10:06:05.148Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8924bc4f-bb78-4e10-9133-feca9422fed7","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:05.148Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8924bc4f-bb78-4e10-9133-feca9422fed7","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:05.148Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8924bc4f-bb78-4e10-9133-feca9422fed7","filter labels":["test-ns-ltx56",{"ray.io/cluster":"submitter-timeout-lgrnm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:06:05.148Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8924bc4f-bb78-4e10-9133-feca9422fed7"}
{"level":"error","ts":"2026-09-18T10:06:05.148Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8924bc4f-bb78-4e10-9133-feca9422fed7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-submitter-timeout-lgrnm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:05.789Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe"}
{"level":"info","ts":"2026-09-18T10:06:05.789Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe"}
{"level":"info","ts":"2026-09-18T10:06:05.789Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe"}
{"level":"info","ts":"2026-09-18T10:06:05.789Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:05.789Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe"}
{"level":"info","ts":"2026-09-18T10:06:05.789Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:06:05.789Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe"}
{"level":"info","ts":"2026-09-18T10:06:05.789Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","mode":"IntegratedOAuth","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:05.789Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","cluster":"submitter-timeout-lgrnm","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:06:05.789Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","cluster":"submitter-timeout-lgrnm","authType":"IntegratedOAuth","serviceAccount":"submitter-timeout-lgrnm-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T10:06:05.789Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"submitter-timeout-lgrnm","ray.io/group":"headgroup","ray.io/identifier":"submitter-timeout-lgrnm-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:06:05.789Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T10:06:05.789Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:06:05.789Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:06:05.789Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","enabled":true}
{"level":"info","ts":"2026-09-18T10:06:05.825Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","name":"submitter-timeout-lgrnm-head-8n6xl"}
{"level":"info","ts":"2026-09-18T10:06:05.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:06:05.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:05.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:06:05.826Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:06:05.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:05.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:06:05.826Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe"}
{"level":"info","ts":"2026-09-18T10:06:05.826Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","nodeType":"worker","rayStartParams":{"address":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:06:05.826Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","rayStartCmd":"ray start  --address=submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:06:05.826Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:06:05.826Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","enabled":true}
{"level":"info","ts":"2026-09-18T10:06:05.869Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","name":"submitter-timeout-lgrnm-small-group-worker-xcz2v"}
{"level":"info","ts":"2026-09-18T10:06:05.870Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:05.870Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"fdd5cd51-a15c-422f-b636-ba70a4dc6ebe","name":"submitter-timeout-lgrnm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:06:05.892Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06"}
{"level":"info","ts":"2026-09-18T10:06:05.892Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06"}
{"level":"info","ts":"2026-09-18T10:06:05.892Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06"}
{"level":"info","ts":"2026-09-18T10:06:05.892Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:05.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:06:05.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:05.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:06:05.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:05.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:05.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:06:05.893Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:06:05.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:05.893Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06","podName":"submitter-timeout-lgrnm-small-group-worker-xcz2v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:05.893Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06","podName":"submitter-timeout-lgrnm-small-group-worker-xcz2v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:05.893Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06","name":"submitter-timeout-lgrnm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:06:05.898Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"781e007a-d7a4-4951-9c1e-c72757b657c3","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:05.898Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"e1a2dd0d-b8f4-4692-833c-0c109dfb9eac","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:05.898Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"e1a2dd0d-b8f4-4692-833c-0c109dfb9eac"}
{"level":"info","ts":"2026-09-18T10:06:05.898Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"26a0b232-f00a-4e68-abb4-97634f947937","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:05.898Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"26a0b232-f00a-4e68-abb4-97634f947937","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:05.898Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"26a0b232-f00a-4e68-abb4-97634f947937","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:05.898Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"26a0b232-f00a-4e68-abb4-97634f947937"}
{"level":"info","ts":"2026-09-18T10:06:05.898Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"26a0b232-f00a-4e68-abb4-97634f947937","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:05.898Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"26a0b232-f00a-4e68-abb4-97634f947937","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:05.898Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"26a0b232-f00a-4e68-abb4-97634f947937","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:06:05.906Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06","name":"submitter-timeout-lgrnm","error":"Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-lgrnm\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm"}}
{"level":"info","ts":"2026-09-18T10:06:05.906Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06"}
{"level":"error","ts":"2026-09-18T10:06:05.906Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a6b53816-8fdb-4c66-bf2a-88dc6fee6e06","error":"Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-lgrnm\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:06:05.911Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8a0b164f-5637-4b14-bf42-a5a1579c0d48"}
{"level":"info","ts":"2026-09-18T10:06:05.911Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8a0b164f-5637-4b14-bf42-a5a1579c0d48"}
{"level":"info","ts":"2026-09-18T10:06:05.911Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8a0b164f-5637-4b14-bf42-a5a1579c0d48"}
{"level":"info","ts":"2026-09-18T10:06:05.911Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8a0b164f-5637-4b14-bf42-a5a1579c0d48","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:05.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8a0b164f-5637-4b14-bf42-a5a1579c0d48","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:06:05.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8a0b164f-5637-4b14-bf42-a5a1579c0d48","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:05.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8a0b164f-5637-4b14-bf42-a5a1579c0d48","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:06:05.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8a0b164f-5637-4b14-bf42-a5a1579c0d48","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:05.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8a0b164f-5637-4b14-bf42-a5a1579c0d48","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:05.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8a0b164f-5637-4b14-bf42-a5a1579c0d48","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:06:05.912Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8a0b164f-5637-4b14-bf42-a5a1579c0d48","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:06:05.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8a0b164f-5637-4b14-bf42-a5a1579c0d48","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:05.912Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8a0b164f-5637-4b14-bf42-a5a1579c0d48","podName":"submitter-timeout-lgrnm-small-group-worker-xcz2v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:05.912Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8a0b164f-5637-4b14-bf42-a5a1579c0d48","podName":"submitter-timeout-lgrnm-small-group-worker-xcz2v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:05.912Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8a0b164f-5637-4b14-bf42-a5a1579c0d48","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:06:05.912Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8a0b164f-5637-4b14-bf42-a5a1579c0d48","seconds":300}
{"level":"info","ts":"2026-09-18T10:06:05.922Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"89907270-81cc-4715-80b3-85069f89e67e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:05.922Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"89907270-81cc-4715-80b3-85069f89e67e","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:05.922Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"89907270-81cc-4715-80b3-85069f89e67e","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:05.922Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"89907270-81cc-4715-80b3-85069f89e67e"}
{"level":"info","ts":"2026-09-18T10:06:05.922Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"89907270-81cc-4715-80b3-85069f89e67e","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:05.922Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"89907270-81cc-4715-80b3-85069f89e67e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:05.923Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a676fb1c-5a07-4f3f-8b1b-39b4db19187a"}
{"level":"info","ts":"2026-09-18T10:06:05.923Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a676fb1c-5a07-4f3f-8b1b-39b4db19187a"}
{"level":"info","ts":"2026-09-18T10:06:05.923Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a676fb1c-5a07-4f3f-8b1b-39b4db19187a"}
{"level":"info","ts":"2026-09-18T10:06:05.923Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a676fb1c-5a07-4f3f-8b1b-39b4db19187a","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:05.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a676fb1c-5a07-4f3f-8b1b-39b4db19187a","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:06:05.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a676fb1c-5a07-4f3f-8b1b-39b4db19187a","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:05.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a676fb1c-5a07-4f3f-8b1b-39b4db19187a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:06:05.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a676fb1c-5a07-4f3f-8b1b-39b4db19187a","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:05.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a676fb1c-5a07-4f3f-8b1b-39b4db19187a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:05.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a676fb1c-5a07-4f3f-8b1b-39b4db19187a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:06:05.923Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a676fb1c-5a07-4f3f-8b1b-39b4db19187a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:06:05.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a676fb1c-5a07-4f3f-8b1b-39b4db19187a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:05.923Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a676fb1c-5a07-4f3f-8b1b-39b4db19187a","podName":"submitter-timeout-lgrnm-small-group-worker-xcz2v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:05.923Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a676fb1c-5a07-4f3f-8b1b-39b4db19187a","podName":"submitter-timeout-lgrnm-small-group-worker-xcz2v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:05.923Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"a676fb1c-5a07-4f3f-8b1b-39b4db19187a","name":"submitter-timeout-lgrnm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:06:05.930Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"781e007a-d7a4-4951-9c1e-c72757b657c3","name":"submitter-timeout-lgrnm-head"}
{"level":"info","ts":"2026-09-18T10:06:05.966Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6f451e08-b88f-40f7-b95a-a76b199ced89","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:05.966Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6f451e08-b88f-40f7-b95a-a76b199ced89","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:05.966Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6f451e08-b88f-40f7-b95a-a76b199ced89","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:05.966Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b4eb495a-efc6-44c5-9d3c-e5d71c66e9a5"}
{"level":"info","ts":"2026-09-18T10:06:05.966Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b4eb495a-efc6-44c5-9d3c-e5d71c66e9a5"}
{"level":"info","ts":"2026-09-18T10:06:05.966Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b4eb495a-efc6-44c5-9d3c-e5d71c66e9a5"}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6f451e08-b88f-40f7-b95a-a76b199ced89"}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b4eb495a-efc6-44c5-9d3c-e5d71c66e9a5","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6f451e08-b88f-40f7-b95a-a76b199ced89","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6f451e08-b88f-40f7-b95a-a76b199ced89","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6f451e08-b88f-40f7-b95a-a76b199ced89","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b4eb495a-efc6-44c5-9d3c-e5d71c66e9a5","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b4eb495a-efc6-44c5-9d3c-e5d71c66e9a5","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b4eb495a-efc6-44c5-9d3c-e5d71c66e9a5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b4eb495a-efc6-44c5-9d3c-e5d71c66e9a5","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b4eb495a-efc6-44c5-9d3c-e5d71c66e9a5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b4eb495a-efc6-44c5-9d3c-e5d71c66e9a5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b4eb495a-efc6-44c5-9d3c-e5d71c66e9a5","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b4eb495a-efc6-44c5-9d3c-e5d71c66e9a5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b4eb495a-efc6-44c5-9d3c-e5d71c66e9a5","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b4eb495a-efc6-44c5-9d3c-e5d71c66e9a5","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"64f41f0a-2b3f-41c4-9a17-9053be4b2841","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b4eb495a-efc6-44c5-9d3c-e5d71c66e9a5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b4eb495a-efc6-44c5-9d3c-e5d71c66e9a5","seconds":300}
{"level":"info","ts":"2026-09-18T10:06:05.967Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"64f41f0a-2b3f-41c4-9a17-9053be4b2841"}
{"level":"info","ts":"2026-09-18T10:06:05.978Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"781e007a-d7a4-4951-9c1e-c72757b657c3","name":"submitter-timeout-lgrnm-workers"}
{"level":"info","ts":"2026-09-18T10:06:05.978Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"781e007a-d7a4-4951-9c1e-c72757b657c3","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:05.978Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"72627843-457d-4c9b-83a8-0c83766f5558","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:05.995Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"52c39f18-b2d5-42ac-a9cd-44b299a7878b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:05.995Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"52c39f18-b2d5-42ac-a9cd-44b299a7878b","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:05.995Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"52c39f18-b2d5-42ac-a9cd-44b299a7878b","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:05.996Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"52c39f18-b2d5-42ac-a9cd-44b299a7878b"}
{"level":"info","ts":"2026-09-18T10:06:05.996Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"52c39f18-b2d5-42ac-a9cd-44b299a7878b","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:05.996Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"52c39f18-b2d5-42ac-a9cd-44b299a7878b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:05.999Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"72627843-457d-4c9b-83a8-0c83766f5558","name":"submitter-timeout-lgrnm-head"}
{"level":"info","ts":"2026-09-18T10:06:06.011Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"72627843-457d-4c9b-83a8-0c83766f5558","name":"submitter-timeout-lgrnm-workers"}
{"level":"info","ts":"2026-09-18T10:06:06.011Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"72627843-457d-4c9b-83a8-0c83766f5558","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:06.126Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-ltx56"},"reconcileID":"2e159ba1-5a7c-4f84-9242-af83cb03d795"}
{"level":"info","ts":"2026-09-18T10:06:06.445Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0d100958-8bb2-4a41-a602-f90cb669e7c2"}
{"level":"info","ts":"2026-09-18T10:06:06.445Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0d100958-8bb2-4a41-a602-f90cb669e7c2"}
{"level":"info","ts":"2026-09-18T10:06:06.445Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0d100958-8bb2-4a41-a602-f90cb669e7c2"}
{"level":"info","ts":"2026-09-18T10:06:06.445Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0d100958-8bb2-4a41-a602-f90cb669e7c2","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:06.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0d100958-8bb2-4a41-a602-f90cb669e7c2","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:06:06.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0d100958-8bb2-4a41-a602-f90cb669e7c2","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:06.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0d100958-8bb2-4a41-a602-f90cb669e7c2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:06:06.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0d100958-8bb2-4a41-a602-f90cb669e7c2","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:06.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0d100958-8bb2-4a41-a602-f90cb669e7c2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:06.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0d100958-8bb2-4a41-a602-f90cb669e7c2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:06:06.445Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0d100958-8bb2-4a41-a602-f90cb669e7c2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:06:06.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0d100958-8bb2-4a41-a602-f90cb669e7c2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:06.445Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0d100958-8bb2-4a41-a602-f90cb669e7c2","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:06.445Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0d100958-8bb2-4a41-a602-f90cb669e7c2","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:06.445Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0d100958-8bb2-4a41-a602-f90cb669e7c2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:06:06.445Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0d100958-8bb2-4a41-a602-f90cb669e7c2","seconds":300}
{"level":"info","ts":"2026-09-18T10:06:06.482Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b39c9200-4130-436e-b5d4-4b36a2cce656"}
{"level":"info","ts":"2026-09-18T10:06:06.482Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b39c9200-4130-436e-b5d4-4b36a2cce656"}
{"level":"info","ts":"2026-09-18T10:06:06.482Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b39c9200-4130-436e-b5d4-4b36a2cce656"}
{"level":"info","ts":"2026-09-18T10:06:06.482Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b39c9200-4130-436e-b5d4-4b36a2cce656","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:06.482Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b39c9200-4130-436e-b5d4-4b36a2cce656","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:06:06.482Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b39c9200-4130-436e-b5d4-4b36a2cce656","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:06.482Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b39c9200-4130-436e-b5d4-4b36a2cce656","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:06:06.482Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b39c9200-4130-436e-b5d4-4b36a2cce656","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:06.482Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b39c9200-4130-436e-b5d4-4b36a2cce656","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:06.482Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b39c9200-4130-436e-b5d4-4b36a2cce656","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:06:06.483Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b39c9200-4130-436e-b5d4-4b36a2cce656","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:06:06.483Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b39c9200-4130-436e-b5d4-4b36a2cce656","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:06.483Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b39c9200-4130-436e-b5d4-4b36a2cce656","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:06.483Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b39c9200-4130-436e-b5d4-4b36a2cce656","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:06.483Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b39c9200-4130-436e-b5d4-4b36a2cce656","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:06:06.483Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"b39c9200-4130-436e-b5d4-4b36a2cce656","seconds":300}
{"level":"info","ts":"2026-09-18T10:06:07.077Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8280411f-1e25-4195-884a-afbe5d5867b9"}
{"level":"info","ts":"2026-09-18T10:06:07.077Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8280411f-1e25-4195-884a-afbe5d5867b9"}
{"level":"info","ts":"2026-09-18T10:06:07.077Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8280411f-1e25-4195-884a-afbe5d5867b9"}
{"level":"info","ts":"2026-09-18T10:06:07.077Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8280411f-1e25-4195-884a-afbe5d5867b9","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:07.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8280411f-1e25-4195-884a-afbe5d5867b9","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:06:07.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8280411f-1e25-4195-884a-afbe5d5867b9","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:07.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8280411f-1e25-4195-884a-afbe5d5867b9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:06:07.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8280411f-1e25-4195-884a-afbe5d5867b9","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:07.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8280411f-1e25-4195-884a-afbe5d5867b9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:07.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8280411f-1e25-4195-884a-afbe5d5867b9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:06:07.078Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8280411f-1e25-4195-884a-afbe5d5867b9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:06:07.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8280411f-1e25-4195-884a-afbe5d5867b9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:07.078Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8280411f-1e25-4195-884a-afbe5d5867b9","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:07.078Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8280411f-1e25-4195-884a-afbe5d5867b9","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:07.078Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8280411f-1e25-4195-884a-afbe5d5867b9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:06:07.078Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"8280411f-1e25-4195-884a-afbe5d5867b9","seconds":300}
{"level":"info","ts":"2026-09-18T10:06:07.226Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c3e58653-010b-40da-87fe-8cfb858c1156","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:07.226Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c3e58653-010b-40da-87fe-8cfb858c1156","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:07.227Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c3e58653-010b-40da-87fe-8cfb858c1156","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:07.227Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c3e58653-010b-40da-87fe-8cfb858c1156"}
{"level":"info","ts":"2026-09-18T10:06:07.227Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c3e58653-010b-40da-87fe-8cfb858c1156","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:07.227Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c3e58653-010b-40da-87fe-8cfb858c1156","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:07.364Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e71c4b32-3534-42ca-9a2e-5946747df098"}
{"level":"info","ts":"2026-09-18T10:06:07.364Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e71c4b32-3534-42ca-9a2e-5946747df098"}
{"level":"info","ts":"2026-09-18T10:06:07.364Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e71c4b32-3534-42ca-9a2e-5946747df098"}
{"level":"info","ts":"2026-09-18T10:06:07.364Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e71c4b32-3534-42ca-9a2e-5946747df098","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:07.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e71c4b32-3534-42ca-9a2e-5946747df098","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:06:07.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e71c4b32-3534-42ca-9a2e-5946747df098","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:07.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e71c4b32-3534-42ca-9a2e-5946747df098","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:06:07.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e71c4b32-3534-42ca-9a2e-5946747df098","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:07.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e71c4b32-3534-42ca-9a2e-5946747df098","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:07.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e71c4b32-3534-42ca-9a2e-5946747df098","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:06:07.364Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e71c4b32-3534-42ca-9a2e-5946747df098","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:06:07.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e71c4b32-3534-42ca-9a2e-5946747df098","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:07.364Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e71c4b32-3534-42ca-9a2e-5946747df098","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:07.364Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e71c4b32-3534-42ca-9a2e-5946747df098","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:07.364Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e71c4b32-3534-42ca-9a2e-5946747df098","name":"submitter-timeout-lgrnm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:06:07.379Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"8ab9aa82-787f-48b2-ae04-39c279127ebe","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:07.379Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b91a123b-1aa4-439c-b850-f27f53666680","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:07.379Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b91a123b-1aa4-439c-b850-f27f53666680","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:07.380Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b91a123b-1aa4-439c-b850-f27f53666680","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:07.380Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"9dedf0fa-e10e-40d2-a184-dda02457203e","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:07.380Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b91a123b-1aa4-439c-b850-f27f53666680"}
{"level":"info","ts":"2026-09-18T10:06:07.380Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b91a123b-1aa4-439c-b850-f27f53666680","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:07.380Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b91a123b-1aa4-439c-b850-f27f53666680","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:07.380Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"b91a123b-1aa4-439c-b850-f27f53666680","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:06:07.379Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"8ab9aa82-787f-48b2-ae04-39c279127ebe","pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","ip":"10.132.0.52"}
{"level":"info","ts":"2026-09-18T10:06:07.380Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"8ab9aa82-787f-48b2-ae04-39c279127ebe","pod":"submitter-timeout-lgrnm-head-8n6xl","ip":"10.133.0.45"}
{"level":"info","ts":"2026-09-18T10:06:07.397Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"9dedf0fa-e10e-40d2-a184-dda02457203e","name":"submitter-timeout-lgrnm-head"}
{"level":"info","ts":"2026-09-18T10:06:07.398Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0880a26f-8365-4eea-8a2b-afbcc7a2a392","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:07.398Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0880a26f-8365-4eea-8a2b-afbcc7a2a392","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:07.398Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0880a26f-8365-4eea-8a2b-afbcc7a2a392","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:07.398Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0880a26f-8365-4eea-8a2b-afbcc7a2a392"}
{"level":"info","ts":"2026-09-18T10:06:07.398Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0880a26f-8365-4eea-8a2b-afbcc7a2a392","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:07.398Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0880a26f-8365-4eea-8a2b-afbcc7a2a392","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:07.406Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"8ab9aa82-787f-48b2-ae04-39c279127ebe","certificate":"ray-head-cert-submitter-timeout-lgrnm","dnsNames":["submitter-timeout-lgrnm-head-svc","localhost","submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc","submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.52","10.133.0.45","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:06:07.412Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"9dedf0fa-e10e-40d2-a184-dda02457203e","name":"submitter-timeout-lgrnm-workers"}
{"level":"info","ts":"2026-09-18T10:06:07.412Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"9dedf0fa-e10e-40d2-a184-dda02457203e","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:07.427Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"8ab9aa82-787f-48b2-ae04-39c279127ebe","certificate":"ray-worker-cert-submitter-timeout-lgrnm","dnsNames":["submitter-timeout-lgrnm-worker-svc","localhost","submitter-timeout-lgrnm-small-group","submitter-timeout-lgrnm-worker-svc.test-ns-ltx56.svc","submitter-timeout-lgrnm-worker-svc.test-ns-ltx56.svc.cluster.local","*.submitter-timeout-lgrnm-worker-svc.test-ns-ltx56.svc","*.submitter-timeout-lgrnm-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.52","10.133.0.45","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:06:07.427Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"8ab9aa82-787f-48b2-ae04-39c279127ebe"}
{"level":"info","ts":"2026-09-18T10:06:07.967Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86e16250-34d2-4bc9-85b9-773e51899ffb"}
{"level":"info","ts":"2026-09-18T10:06:07.967Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86e16250-34d2-4bc9-85b9-773e51899ffb"}
{"level":"info","ts":"2026-09-18T10:06:07.967Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86e16250-34d2-4bc9-85b9-773e51899ffb"}
{"level":"info","ts":"2026-09-18T10:06:07.967Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86e16250-34d2-4bc9-85b9-773e51899ffb","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:06:07.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86e16250-34d2-4bc9-85b9-773e51899ffb","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:06:07.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86e16250-34d2-4bc9-85b9-773e51899ffb","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:07.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86e16250-34d2-4bc9-85b9-773e51899ffb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:06:07.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86e16250-34d2-4bc9-85b9-773e51899ffb","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:06:07.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86e16250-34d2-4bc9-85b9-773e51899ffb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:07.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86e16250-34d2-4bc9-85b9-773e51899ffb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:06:07.967Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86e16250-34d2-4bc9-85b9-773e51899ffb","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:06:07.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86e16250-34d2-4bc9-85b9-773e51899ffb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:06:07.967Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86e16250-34d2-4bc9-85b9-773e51899ffb","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:07.967Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86e16250-34d2-4bc9-85b9-773e51899ffb","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:06:07.967Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86e16250-34d2-4bc9-85b9-773e51899ffb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:06:07.967Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86e16250-34d2-4bc9-85b9-773e51899ffb","seconds":300}
{"level":"info","ts":"2026-09-18T10:06:10.227Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6620f806-a26f-4d51-9394-19669745faf8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:10.227Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6620f806-a26f-4d51-9394-19669745faf8","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:10.227Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6620f806-a26f-4d51-9394-19669745faf8","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:10.227Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6620f806-a26f-4d51-9394-19669745faf8"}
{"level":"info","ts":"2026-09-18T10:06:10.228Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6620f806-a26f-4d51-9394-19669745faf8","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:10.228Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6620f806-a26f-4d51-9394-19669745faf8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:13.228Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"423e9c92-c2c6-4e15-a4b1-d15d95e0cbf2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:13.228Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"423e9c92-c2c6-4e15-a4b1-d15d95e0cbf2","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:13.228Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"423e9c92-c2c6-4e15-a4b1-d15d95e0cbf2","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:13.228Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"423e9c92-c2c6-4e15-a4b1-d15d95e0cbf2"}
{"level":"info","ts":"2026-09-18T10:06:13.228Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"423e9c92-c2c6-4e15-a4b1-d15d95e0cbf2","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:13.228Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"423e9c92-c2c6-4e15-a4b1-d15d95e0cbf2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:16.229Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0c7e4da6-8871-4833-8d74-e2cc8d4ac42b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:16.229Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0c7e4da6-8871-4833-8d74-e2cc8d4ac42b","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:16.229Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0c7e4da6-8871-4833-8d74-e2cc8d4ac42b","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:16.229Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0c7e4da6-8871-4833-8d74-e2cc8d4ac42b"}
{"level":"info","ts":"2026-09-18T10:06:16.229Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0c7e4da6-8871-4833-8d74-e2cc8d4ac42b","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:16.229Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0c7e4da6-8871-4833-8d74-e2cc8d4ac42b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:19.230Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d89d08b7-b96a-4532-8526-482f3536d10c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:19.230Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d89d08b7-b96a-4532-8526-482f3536d10c","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:19.230Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d89d08b7-b96a-4532-8526-482f3536d10c","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:19.230Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d89d08b7-b96a-4532-8526-482f3536d10c"}
{"level":"info","ts":"2026-09-18T10:06:19.230Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d89d08b7-b96a-4532-8526-482f3536d10c","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:19.230Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d89d08b7-b96a-4532-8526-482f3536d10c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:22.231Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"309f7a99-9fd0-4d97-909d-a1456c2ce7dc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:22.231Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"309f7a99-9fd0-4d97-909d-a1456c2ce7dc","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:22.231Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"309f7a99-9fd0-4d97-909d-a1456c2ce7dc","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:22.231Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"309f7a99-9fd0-4d97-909d-a1456c2ce7dc"}
{"level":"info","ts":"2026-09-18T10:06:22.231Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"309f7a99-9fd0-4d97-909d-a1456c2ce7dc","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:22.231Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"309f7a99-9fd0-4d97-909d-a1456c2ce7dc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:25.231Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5ff0f30b-2881-44c0-bc7f-54c6c687c91e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:25.231Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5ff0f30b-2881-44c0-bc7f-54c6c687c91e","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:25.231Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5ff0f30b-2881-44c0-bc7f-54c6c687c91e","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:25.231Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5ff0f30b-2881-44c0-bc7f-54c6c687c91e"}
{"level":"info","ts":"2026-09-18T10:06:25.231Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5ff0f30b-2881-44c0-bc7f-54c6c687c91e","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:25.231Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5ff0f30b-2881-44c0-bc7f-54c6c687c91e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:28.233Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f134fc43-5bbe-498a-aecb-046b2bd2ef53","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:28.233Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f134fc43-5bbe-498a-aecb-046b2bd2ef53","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:28.233Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f134fc43-5bbe-498a-aecb-046b2bd2ef53","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:28.233Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f134fc43-5bbe-498a-aecb-046b2bd2ef53"}
{"level":"info","ts":"2026-09-18T10:06:28.233Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f134fc43-5bbe-498a-aecb-046b2bd2ef53","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:28.233Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f134fc43-5bbe-498a-aecb-046b2bd2ef53","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:31.234Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"96d028ad-b28c-4469-81ea-5b2a8b6d86e1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:31.234Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"96d028ad-b28c-4469-81ea-5b2a8b6d86e1","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:31.234Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"96d028ad-b28c-4469-81ea-5b2a8b6d86e1","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:31.234Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"96d028ad-b28c-4469-81ea-5b2a8b6d86e1"}
{"level":"info","ts":"2026-09-18T10:06:31.234Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"96d028ad-b28c-4469-81ea-5b2a8b6d86e1","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:31.234Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"96d028ad-b28c-4469-81ea-5b2a8b6d86e1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:34.234Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"9f8a95e0-9104-4b0d-89a5-682d86ddcbf9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:34.234Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"9f8a95e0-9104-4b0d-89a5-682d86ddcbf9","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:34.234Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"9f8a95e0-9104-4b0d-89a5-682d86ddcbf9","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:34.235Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"9f8a95e0-9104-4b0d-89a5-682d86ddcbf9"}
{"level":"info","ts":"2026-09-18T10:06:34.235Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"9f8a95e0-9104-4b0d-89a5-682d86ddcbf9","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:34.235Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"9f8a95e0-9104-4b0d-89a5-682d86ddcbf9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:34.668Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"9fedadb4-7a96-42ea-a938-464265d1140d","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:34.668Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"9fedadb4-7a96-42ea-a938-464265d1140d","pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","ip":"10.132.0.52"}
{"level":"info","ts":"2026-09-18T10:06:34.668Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"9fedadb4-7a96-42ea-a938-464265d1140d","pod":"submitter-timeout-lgrnm-head-8n6xl","ip":"10.133.0.45"}
{"level":"info","ts":"2026-09-18T10:06:34.668Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"9fedadb4-7a96-42ea-a938-464265d1140d","rayCluster":"submitter-timeout-lgrnm","podIPs":["10.132.0.52","10.133.0.45"]}
{"level":"info","ts":"2026-09-18T10:06:37.236Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0626fb5c-1417-4db0-9a00-7ebe40c133e3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:37.236Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0626fb5c-1417-4db0-9a00-7ebe40c133e3","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:37.236Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0626fb5c-1417-4db0-9a00-7ebe40c133e3","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:37.236Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0626fb5c-1417-4db0-9a00-7ebe40c133e3"}
{"level":"info","ts":"2026-09-18T10:06:37.236Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0626fb5c-1417-4db0-9a00-7ebe40c133e3","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:37.236Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0626fb5c-1417-4db0-9a00-7ebe40c133e3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:40.237Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"06981322-7edd-48ee-839d-ef4bc0ed7f0c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:40.237Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"06981322-7edd-48ee-839d-ef4bc0ed7f0c","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:40.237Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"06981322-7edd-48ee-839d-ef4bc0ed7f0c","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:40.237Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"06981322-7edd-48ee-839d-ef4bc0ed7f0c"}
{"level":"info","ts":"2026-09-18T10:06:40.237Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"06981322-7edd-48ee-839d-ef4bc0ed7f0c","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:40.237Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"06981322-7edd-48ee-839d-ef4bc0ed7f0c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:43.238Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"bb1bf79d-2190-46f9-8c6e-2ad7c6bcce1d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:43.238Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"bb1bf79d-2190-46f9-8c6e-2ad7c6bcce1d","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:43.238Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"bb1bf79d-2190-46f9-8c6e-2ad7c6bcce1d","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:43.238Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"bb1bf79d-2190-46f9-8c6e-2ad7c6bcce1d"}
{"level":"info","ts":"2026-09-18T10:06:43.238Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"bb1bf79d-2190-46f9-8c6e-2ad7c6bcce1d","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:43.238Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"bb1bf79d-2190-46f9-8c6e-2ad7c6bcce1d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:45.580Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"29b1d4ad-399b-4912-8275-bf9992bb055f","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:06:45.580Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"29b1d4ad-399b-4912-8275-bf9992bb055f","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:06:45.580Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"29b1d4ad-399b-4912-8275-bf9992bb055f","pod":"long-running-5qfl8-small-group-worker-tbqtj","ip":"10.132.0.47"}
{"level":"info","ts":"2026-09-18T10:06:45.580Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"29b1d4ad-399b-4912-8275-bf9992bb055f","rayCluster":"long-running-5qfl8","podIPs":["10.133.0.42","10.132.0.47"]}
{"level":"info","ts":"2026-09-18T10:06:46.239Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"da5532e3-5ac9-4f76-b213-9e70ea4af3de","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:46.239Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"da5532e3-5ac9-4f76-b213-9e70ea4af3de","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:46.239Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"da5532e3-5ac9-4f76-b213-9e70ea4af3de","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:46.239Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"da5532e3-5ac9-4f76-b213-9e70ea4af3de"}
{"level":"info","ts":"2026-09-18T10:06:46.239Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"da5532e3-5ac9-4f76-b213-9e70ea4af3de","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:46.239Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"da5532e3-5ac9-4f76-b213-9e70ea4af3de","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:49.239Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"dfe9e6f6-efa4-4d09-ad32-6b4b0fc16503","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:49.239Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"dfe9e6f6-efa4-4d09-ad32-6b4b0fc16503","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:49.239Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"dfe9e6f6-efa4-4d09-ad32-6b4b0fc16503","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:49.240Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"dfe9e6f6-efa4-4d09-ad32-6b4b0fc16503"}
{"level":"info","ts":"2026-09-18T10:06:49.240Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"dfe9e6f6-efa4-4d09-ad32-6b4b0fc16503","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:49.240Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"dfe9e6f6-efa4-4d09-ad32-6b4b0fc16503","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:51.506Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"ddf9d789-60c2-4ef4-985d-426e5599c45c","namespace":"test-ns-ltx56","clusterName":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:51.507Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-r4xnr","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"delete-head-after-submit-r4xnr","reconcileID":"ddf9d789-60c2-4ef4-985d-426e5599c45c","namespace":"test-ns-ltx56","clusterName":"delete-head-after-submit-r4xnr"}
{"level":"info","ts":"2026-09-18T10:06:52.240Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"74b50fe4-742f-4a17-8f06-07b0fb90f0dc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:52.240Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"74b50fe4-742f-4a17-8f06-07b0fb90f0dc","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:52.240Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"74b50fe4-742f-4a17-8f06-07b0fb90f0dc","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:52.240Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"74b50fe4-742f-4a17-8f06-07b0fb90f0dc"}
{"level":"info","ts":"2026-09-18T10:06:52.240Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"74b50fe4-742f-4a17-8f06-07b0fb90f0dc","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:52.240Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"74b50fe4-742f-4a17-8f06-07b0fb90f0dc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:55.241Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"09d9f06a-96b1-492d-bede-9d103da3cb22","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:55.241Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"09d9f06a-96b1-492d-bede-9d103da3cb22","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:55.241Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"09d9f06a-96b1-492d-bede-9d103da3cb22","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:55.241Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"09d9f06a-96b1-492d-bede-9d103da3cb22"}
{"level":"info","ts":"2026-09-18T10:06:55.241Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"09d9f06a-96b1-492d-bede-9d103da3cb22","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:55.241Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"09d9f06a-96b1-492d-bede-9d103da3cb22","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:06:58.242Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5a7f1ba9-314d-42d1-b361-21c2c03386c2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:06:58.242Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5a7f1ba9-314d-42d1-b361-21c2c03386c2","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:06:58.242Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5a7f1ba9-314d-42d1-b361-21c2c03386c2","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:06:58.242Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5a7f1ba9-314d-42d1-b361-21c2c03386c2"}
{"level":"info","ts":"2026-09-18T10:06:58.242Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5a7f1ba9-314d-42d1-b361-21c2c03386c2","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:06:58.242Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5a7f1ba9-314d-42d1-b361-21c2c03386c2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:01.243Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d1f13627-a3ec-4fb6-a0d9-5d8e91933fd5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:01.243Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d1f13627-a3ec-4fb6-a0d9-5d8e91933fd5","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:01.243Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d1f13627-a3ec-4fb6-a0d9-5d8e91933fd5","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:01.243Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d1f13627-a3ec-4fb6-a0d9-5d8e91933fd5"}
{"level":"info","ts":"2026-09-18T10:07:01.243Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d1f13627-a3ec-4fb6-a0d9-5d8e91933fd5","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:01.243Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d1f13627-a3ec-4fb6-a0d9-5d8e91933fd5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:04.244Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"660ec4ad-1edf-4a37-962f-0a7a3affc7e4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:04.244Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"660ec4ad-1edf-4a37-962f-0a7a3affc7e4","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:04.244Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"660ec4ad-1edf-4a37-962f-0a7a3affc7e4","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:04.244Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"660ec4ad-1edf-4a37-962f-0a7a3affc7e4"}
{"level":"info","ts":"2026-09-18T10:07:04.244Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"660ec4ad-1edf-4a37-962f-0a7a3affc7e4","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:04.244Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"660ec4ad-1edf-4a37-962f-0a7a3affc7e4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:07.245Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"29d5b0e5-ea89-4a64-a541-10670799be44","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:07.245Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"29d5b0e5-ea89-4a64-a541-10670799be44","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:07.246Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"29d5b0e5-ea89-4a64-a541-10670799be44","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:07.246Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"29d5b0e5-ea89-4a64-a541-10670799be44"}
{"level":"info","ts":"2026-09-18T10:07:07.246Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"29d5b0e5-ea89-4a64-a541-10670799be44","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:07.246Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"29d5b0e5-ea89-4a64-a541-10670799be44","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:10.246Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"22fc3e27-b519-4541-89c4-fc703f925965","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:10.246Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"22fc3e27-b519-4541-89c4-fc703f925965","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:10.246Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"22fc3e27-b519-4541-89c4-fc703f925965","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:10.246Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"22fc3e27-b519-4541-89c4-fc703f925965"}
{"level":"info","ts":"2026-09-18T10:07:10.246Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"22fc3e27-b519-4541-89c4-fc703f925965","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:10.246Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"22fc3e27-b519-4541-89c4-fc703f925965","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:13.247Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"23db368e-0cb8-4b75-bfc8-d67a2f8921b2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:13.247Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"23db368e-0cb8-4b75-bfc8-d67a2f8921b2","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:13.247Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"23db368e-0cb8-4b75-bfc8-d67a2f8921b2","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:13.247Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"23db368e-0cb8-4b75-bfc8-d67a2f8921b2"}
{"level":"info","ts":"2026-09-18T10:07:13.247Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"23db368e-0cb8-4b75-bfc8-d67a2f8921b2","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:13.247Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"23db368e-0cb8-4b75-bfc8-d67a2f8921b2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:16.248Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"eea68167-7846-40a6-a1ba-6c180e7faeb3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:16.248Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"eea68167-7846-40a6-a1ba-6c180e7faeb3","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:16.248Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"eea68167-7846-40a6-a1ba-6c180e7faeb3","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:16.248Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"eea68167-7846-40a6-a1ba-6c180e7faeb3"}
{"level":"info","ts":"2026-09-18T10:07:16.248Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"eea68167-7846-40a6-a1ba-6c180e7faeb3","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:16.248Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"eea68167-7846-40a6-a1ba-6c180e7faeb3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:17.417Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"21d55a90-ce44-4d1e-b882-1843a280be7e"}
{"level":"info","ts":"2026-09-18T10:07:17.417Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"21d55a90-ce44-4d1e-b882-1843a280be7e"}
{"level":"info","ts":"2026-09-18T10:07:17.417Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"21d55a90-ce44-4d1e-b882-1843a280be7e"}
{"level":"info","ts":"2026-09-18T10:07:17.417Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"21d55a90-ce44-4d1e-b882-1843a280be7e","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:07:17.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"21d55a90-ce44-4d1e-b882-1843a280be7e","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:07:17.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"21d55a90-ce44-4d1e-b882-1843a280be7e","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:17.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"21d55a90-ce44-4d1e-b882-1843a280be7e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:07:17.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"21d55a90-ce44-4d1e-b882-1843a280be7e","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:17.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"21d55a90-ce44-4d1e-b882-1843a280be7e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:17.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"21d55a90-ce44-4d1e-b882-1843a280be7e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:07:17.417Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"21d55a90-ce44-4d1e-b882-1843a280be7e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:07:17.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"21d55a90-ce44-4d1e-b882-1843a280be7e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:17.418Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"21d55a90-ce44-4d1e-b882-1843a280be7e","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:07:17.418Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"21d55a90-ce44-4d1e-b882-1843a280be7e","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:07:17.418Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"21d55a90-ce44-4d1e-b882-1843a280be7e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:07:17.418Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"21d55a90-ce44-4d1e-b882-1843a280be7e","seconds":300}
{"level":"info","ts":"2026-09-18T10:07:18.414Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86b11a4a-e6c3-4d7b-a768-aa12f92002db"}
{"level":"info","ts":"2026-09-18T10:07:18.414Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86b11a4a-e6c3-4d7b-a768-aa12f92002db"}
{"level":"info","ts":"2026-09-18T10:07:18.414Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86b11a4a-e6c3-4d7b-a768-aa12f92002db"}
{"level":"info","ts":"2026-09-18T10:07:18.414Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86b11a4a-e6c3-4d7b-a768-aa12f92002db","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:07:18.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86b11a4a-e6c3-4d7b-a768-aa12f92002db","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:07:18.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86b11a4a-e6c3-4d7b-a768-aa12f92002db","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:18.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86b11a4a-e6c3-4d7b-a768-aa12f92002db","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:07:18.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86b11a4a-e6c3-4d7b-a768-aa12f92002db","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:18.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86b11a4a-e6c3-4d7b-a768-aa12f92002db","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:18.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86b11a4a-e6c3-4d7b-a768-aa12f92002db","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:07:18.414Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86b11a4a-e6c3-4d7b-a768-aa12f92002db","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:07:18.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86b11a4a-e6c3-4d7b-a768-aa12f92002db","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:18.414Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86b11a4a-e6c3-4d7b-a768-aa12f92002db","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:07:18.414Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86b11a4a-e6c3-4d7b-a768-aa12f92002db","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:07:18.414Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86b11a4a-e6c3-4d7b-a768-aa12f92002db","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:07:18.414Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"86b11a4a-e6c3-4d7b-a768-aa12f92002db","seconds":300}
{"level":"info","ts":"2026-09-18T10:07:19.249Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fec3f4c9-31b9-4f72-82a4-52027adcf31d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:19.249Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fec3f4c9-31b9-4f72-82a4-52027adcf31d","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:19.249Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fec3f4c9-31b9-4f72-82a4-52027adcf31d","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:19.249Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fec3f4c9-31b9-4f72-82a4-52027adcf31d"}
{"level":"info","ts":"2026-09-18T10:07:19.249Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fec3f4c9-31b9-4f72-82a4-52027adcf31d","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:19.249Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fec3f4c9-31b9-4f72-82a4-52027adcf31d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:22.251Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"19aec0e4-813a-4cc4-a1b0-64eaa702714e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:22.251Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"19aec0e4-813a-4cc4-a1b0-64eaa702714e","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:22.251Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"19aec0e4-813a-4cc4-a1b0-64eaa702714e","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:22.251Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"19aec0e4-813a-4cc4-a1b0-64eaa702714e"}
{"level":"info","ts":"2026-09-18T10:07:22.251Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"19aec0e4-813a-4cc4-a1b0-64eaa702714e","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:22.251Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"19aec0e4-813a-4cc4-a1b0-64eaa702714e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:25.251Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d15bdef1-0f73-4d31-aeb4-6e356aee2162","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:25.251Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d15bdef1-0f73-4d31-aeb4-6e356aee2162","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:25.251Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d15bdef1-0f73-4d31-aeb4-6e356aee2162","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:25.251Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d15bdef1-0f73-4d31-aeb4-6e356aee2162"}
{"level":"info","ts":"2026-09-18T10:07:25.251Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d15bdef1-0f73-4d31-aeb4-6e356aee2162","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:25.251Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d15bdef1-0f73-4d31-aeb4-6e356aee2162","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:28.252Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f0a4bb03-e64f-40c3-99a5-1019b2b472c9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:28.252Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f0a4bb03-e64f-40c3-99a5-1019b2b472c9","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:28.252Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f0a4bb03-e64f-40c3-99a5-1019b2b472c9","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:28.252Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f0a4bb03-e64f-40c3-99a5-1019b2b472c9"}
{"level":"info","ts":"2026-09-18T10:07:28.252Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f0a4bb03-e64f-40c3-99a5-1019b2b472c9","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:28.252Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f0a4bb03-e64f-40c3-99a5-1019b2b472c9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:31.253Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6c5895a4-999a-4b98-8a45-837fe9a3a094","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:31.253Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6c5895a4-999a-4b98-8a45-837fe9a3a094","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:31.253Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6c5895a4-999a-4b98-8a45-837fe9a3a094","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:31.253Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6c5895a4-999a-4b98-8a45-837fe9a3a094"}
{"level":"info","ts":"2026-09-18T10:07:31.253Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6c5895a4-999a-4b98-8a45-837fe9a3a094","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:31.253Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6c5895a4-999a-4b98-8a45-837fe9a3a094","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:32.731Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"73643fe3-01ed-41ec-99c8-9bb32e8e6a5b"}
{"level":"info","ts":"2026-09-18T10:07:32.732Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"73643fe3-01ed-41ec-99c8-9bb32e8e6a5b"}
{"level":"info","ts":"2026-09-18T10:07:32.732Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"73643fe3-01ed-41ec-99c8-9bb32e8e6a5b"}
{"level":"info","ts":"2026-09-18T10:07:32.732Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"73643fe3-01ed-41ec-99c8-9bb32e8e6a5b","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:07:32.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"73643fe3-01ed-41ec-99c8-9bb32e8e6a5b","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:07:32.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"73643fe3-01ed-41ec-99c8-9bb32e8e6a5b","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:32.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"73643fe3-01ed-41ec-99c8-9bb32e8e6a5b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:07:32.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"73643fe3-01ed-41ec-99c8-9bb32e8e6a5b","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:32.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"73643fe3-01ed-41ec-99c8-9bb32e8e6a5b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:32.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"73643fe3-01ed-41ec-99c8-9bb32e8e6a5b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:07:32.732Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"73643fe3-01ed-41ec-99c8-9bb32e8e6a5b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:07:32.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"73643fe3-01ed-41ec-99c8-9bb32e8e6a5b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:32.732Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"73643fe3-01ed-41ec-99c8-9bb32e8e6a5b","podName":"submitter-timeout-lgrnm-small-group-worker-xcz2v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:07:32.732Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"73643fe3-01ed-41ec-99c8-9bb32e8e6a5b","podName":"submitter-timeout-lgrnm-small-group-worker-xcz2v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:07:32.732Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"73643fe3-01ed-41ec-99c8-9bb32e8e6a5b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:07:32.732Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"73643fe3-01ed-41ec-99c8-9bb32e8e6a5b","seconds":300}
{"level":"info","ts":"2026-09-18T10:07:33.738Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"3e9d5a63-c3e1-4413-9ec7-4024f0174cb2"}
{"level":"info","ts":"2026-09-18T10:07:33.738Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"3e9d5a63-c3e1-4413-9ec7-4024f0174cb2"}
{"level":"info","ts":"2026-09-18T10:07:33.738Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"3e9d5a63-c3e1-4413-9ec7-4024f0174cb2"}
{"level":"info","ts":"2026-09-18T10:07:33.738Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"3e9d5a63-c3e1-4413-9ec7-4024f0174cb2","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:07:33.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"3e9d5a63-c3e1-4413-9ec7-4024f0174cb2","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:07:33.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"3e9d5a63-c3e1-4413-9ec7-4024f0174cb2","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:33.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"3e9d5a63-c3e1-4413-9ec7-4024f0174cb2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:07:33.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"3e9d5a63-c3e1-4413-9ec7-4024f0174cb2","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:33.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"3e9d5a63-c3e1-4413-9ec7-4024f0174cb2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:33.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"3e9d5a63-c3e1-4413-9ec7-4024f0174cb2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:07:33.738Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"3e9d5a63-c3e1-4413-9ec7-4024f0174cb2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:07:33.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"3e9d5a63-c3e1-4413-9ec7-4024f0174cb2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:33.739Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"3e9d5a63-c3e1-4413-9ec7-4024f0174cb2","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:06:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:07:33.739Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"3e9d5a63-c3e1-4413-9ec7-4024f0174cb2","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:06:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:07:33.739Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"3e9d5a63-c3e1-4413-9ec7-4024f0174cb2","name":"submitter-timeout-lgrnm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:07:33.754Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"b71d51e5-603f-4428-a517-a9ca0ef79621","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:33.754Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"61d498ca-91bf-4164-90cf-c22f247cf644","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:33.754Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"61d498ca-91bf-4164-90cf-c22f247cf644","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:33.754Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"61d498ca-91bf-4164-90cf-c22f247cf644","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:33.754Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"61d498ca-91bf-4164-90cf-c22f247cf644"}
{"level":"info","ts":"2026-09-18T10:07:33.754Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"61d498ca-91bf-4164-90cf-c22f247cf644","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:33.754Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"61d498ca-91bf-4164-90cf-c22f247cf644","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:06:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:33.754Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"61d498ca-91bf-4164-90cf-c22f247cf644","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:07:33.754Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"cc0159b5-54a8-4dcf-801c-f931e83ba657","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:33.754Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"cc0159b5-54a8-4dcf-801c-f931e83ba657","pod":"submitter-timeout-lgrnm-head-8n6xl","ip":"10.133.0.45"}
{"level":"info","ts":"2026-09-18T10:07:33.754Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"cc0159b5-54a8-4dcf-801c-f931e83ba657","pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","ip":"10.132.0.52"}
{"level":"info","ts":"2026-09-18T10:07:33.754Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"cc0159b5-54a8-4dcf-801c-f931e83ba657","rayCluster":"submitter-timeout-lgrnm","podIPs":["10.133.0.45","10.132.0.52"]}
{"level":"info","ts":"2026-09-18T10:07:33.775Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"80f68e56-0ff8-45bb-bf78-60790f3c4731","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:33.775Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"80f68e56-0ff8-45bb-bf78-60790f3c4731","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:33.775Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"80f68e56-0ff8-45bb-bf78-60790f3c4731","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:33.775Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"80f68e56-0ff8-45bb-bf78-60790f3c4731"}
{"level":"info","ts":"2026-09-18T10:07:33.775Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"80f68e56-0ff8-45bb-bf78-60790f3c4731","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:33.775Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"80f68e56-0ff8-45bb-bf78-60790f3c4731","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:33.780Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"b71d51e5-603f-4428-a517-a9ca0ef79621","name":"submitter-timeout-lgrnm-head"}
{"level":"info","ts":"2026-09-18T10:07:33.798Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"b71d51e5-603f-4428-a517-a9ca0ef79621","name":"submitter-timeout-lgrnm-workers"}
{"level":"info","ts":"2026-09-18T10:07:33.798Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"b71d51e5-603f-4428-a517-a9ca0ef79621","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:34.253Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"3d32373d-67b5-466c-8cd5-42be292af5b2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:34.253Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"3d32373d-67b5-466c-8cd5-42be292af5b2","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:34.253Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"3d32373d-67b5-466c-8cd5-42be292af5b2","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:34.253Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"3d32373d-67b5-466c-8cd5-42be292af5b2"}
{"level":"info","ts":"2026-09-18T10:07:34.253Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"3d32373d-67b5-466c-8cd5-42be292af5b2","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:34.253Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"3d32373d-67b5-466c-8cd5-42be292af5b2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:34.669Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"aeba4a8f-d01b-4792-afa8-7bc52399ac93","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:34.669Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"aeba4a8f-d01b-4792-afa8-7bc52399ac93","pod":"submitter-timeout-lgrnm-head-8n6xl","ip":"10.133.0.45"}
{"level":"info","ts":"2026-09-18T10:07:34.669Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"aeba4a8f-d01b-4792-afa8-7bc52399ac93","pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","ip":"10.132.0.52"}
{"level":"info","ts":"2026-09-18T10:07:34.669Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"aeba4a8f-d01b-4792-afa8-7bc52399ac93","rayCluster":"submitter-timeout-lgrnm","podIPs":["10.133.0.45","10.132.0.52"]}
{"level":"info","ts":"2026-09-18T10:07:35.756Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"de9ea146-364e-4fb8-98cf-2a3e09dddfb4"}
{"level":"info","ts":"2026-09-18T10:07:35.756Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"de9ea146-364e-4fb8-98cf-2a3e09dddfb4"}
{"level":"info","ts":"2026-09-18T10:07:35.756Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"de9ea146-364e-4fb8-98cf-2a3e09dddfb4"}
{"level":"info","ts":"2026-09-18T10:07:35.756Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"de9ea146-364e-4fb8-98cf-2a3e09dddfb4","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:07:35.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"de9ea146-364e-4fb8-98cf-2a3e09dddfb4","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:07:35.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"de9ea146-364e-4fb8-98cf-2a3e09dddfb4","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:35.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"de9ea146-364e-4fb8-98cf-2a3e09dddfb4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:07:35.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"de9ea146-364e-4fb8-98cf-2a3e09dddfb4","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:35.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"de9ea146-364e-4fb8-98cf-2a3e09dddfb4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:35.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"de9ea146-364e-4fb8-98cf-2a3e09dddfb4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:07:35.756Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"de9ea146-364e-4fb8-98cf-2a3e09dddfb4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:07:35.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"de9ea146-364e-4fb8-98cf-2a3e09dddfb4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:35.756Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"de9ea146-364e-4fb8-98cf-2a3e09dddfb4","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:06:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:07:35.756Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"de9ea146-364e-4fb8-98cf-2a3e09dddfb4","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:06:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:07:35.757Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"de9ea146-364e-4fb8-98cf-2a3e09dddfb4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:07:35.757Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"de9ea146-364e-4fb8-98cf-2a3e09dddfb4","seconds":300}
{"level":"info","ts":"2026-09-18T10:07:37.254Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6e428d71-0aad-45b1-867e-b18eb5eaac93","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:37.254Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6e428d71-0aad-45b1-867e-b18eb5eaac93","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:37.254Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6e428d71-0aad-45b1-867e-b18eb5eaac93","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:37.254Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6e428d71-0aad-45b1-867e-b18eb5eaac93"}
{"level":"info","ts":"2026-09-18T10:07:37.254Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6e428d71-0aad-45b1-867e-b18eb5eaac93","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:37.254Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"6e428d71-0aad-45b1-867e-b18eb5eaac93","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:38.511Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"d73a3028-91cc-485c-ae71-c8cd8ec603d1"}
{"level":"info","ts":"2026-09-18T10:07:38.511Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"d73a3028-91cc-485c-ae71-c8cd8ec603d1"}
{"level":"info","ts":"2026-09-18T10:07:38.511Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"d73a3028-91cc-485c-ae71-c8cd8ec603d1"}
{"level":"info","ts":"2026-09-18T10:07:38.511Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"d73a3028-91cc-485c-ae71-c8cd8ec603d1","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:07:38.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"d73a3028-91cc-485c-ae71-c8cd8ec603d1","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:07:38.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"d73a3028-91cc-485c-ae71-c8cd8ec603d1","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:38.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"d73a3028-91cc-485c-ae71-c8cd8ec603d1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:07:38.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"d73a3028-91cc-485c-ae71-c8cd8ec603d1","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:38.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"d73a3028-91cc-485c-ae71-c8cd8ec603d1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:38.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"d73a3028-91cc-485c-ae71-c8cd8ec603d1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:07:38.511Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"d73a3028-91cc-485c-ae71-c8cd8ec603d1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:07:38.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"d73a3028-91cc-485c-ae71-c8cd8ec603d1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:38.511Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"d73a3028-91cc-485c-ae71-c8cd8ec603d1","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:06:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:07:38.511Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"d73a3028-91cc-485c-ae71-c8cd8ec603d1","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:06:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:07:38.511Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"d73a3028-91cc-485c-ae71-c8cd8ec603d1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:07:38.511Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"d73a3028-91cc-485c-ae71-c8cd8ec603d1","seconds":300}
{"level":"info","ts":"2026-09-18T10:07:39.515Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"108c7759-5190-43ca-88b9-387854eba9f8"}
{"level":"info","ts":"2026-09-18T10:07:39.515Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"108c7759-5190-43ca-88b9-387854eba9f8"}
{"level":"info","ts":"2026-09-18T10:07:39.515Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"108c7759-5190-43ca-88b9-387854eba9f8"}
{"level":"info","ts":"2026-09-18T10:07:39.515Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"108c7759-5190-43ca-88b9-387854eba9f8","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:07:39.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"108c7759-5190-43ca-88b9-387854eba9f8","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:07:39.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"108c7759-5190-43ca-88b9-387854eba9f8","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:39.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"108c7759-5190-43ca-88b9-387854eba9f8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:07:39.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"108c7759-5190-43ca-88b9-387854eba9f8","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:39.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"108c7759-5190-43ca-88b9-387854eba9f8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:39.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"108c7759-5190-43ca-88b9-387854eba9f8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:07:39.515Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"108c7759-5190-43ca-88b9-387854eba9f8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:07:39.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"108c7759-5190-43ca-88b9-387854eba9f8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:39.516Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"108c7759-5190-43ca-88b9-387854eba9f8","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:06:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:07:39.516Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"108c7759-5190-43ca-88b9-387854eba9f8","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:06:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:07:39.516Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"108c7759-5190-43ca-88b9-387854eba9f8","name":"submitter-timeout-lgrnm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:07:39.530Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"92f1493e-c46e-4e6c-9ee9-6ef699256daf","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:39.530Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fc39b3ba-f12e-4292-8257-074726f19b71","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:39.530Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fc39b3ba-f12e-4292-8257-074726f19b71","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:39.530Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fc39b3ba-f12e-4292-8257-074726f19b71","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:39.530Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"3aab8153-6c47-4a1b-a677-331650b414aa","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:39.530Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"3aab8153-6c47-4a1b-a677-331650b414aa","pod":"submitter-timeout-lgrnm-head-8n6xl","ip":"10.133.0.45"}
{"level":"info","ts":"2026-09-18T10:07:39.531Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"3aab8153-6c47-4a1b-a677-331650b414aa","pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","ip":"10.132.0.52"}
{"level":"info","ts":"2026-09-18T10:07:39.531Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"3aab8153-6c47-4a1b-a677-331650b414aa","rayCluster":"submitter-timeout-lgrnm","podIPs":["10.133.0.45","10.132.0.52"]}
{"level":"info","ts":"2026-09-18T10:07:39.530Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fc39b3ba-f12e-4292-8257-074726f19b71"}
{"level":"info","ts":"2026-09-18T10:07:39.531Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fc39b3ba-f12e-4292-8257-074726f19b71","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:39.531Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fc39b3ba-f12e-4292-8257-074726f19b71","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:39.531Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fc39b3ba-f12e-4292-8257-074726f19b71","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:07:39.548Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"92f1493e-c46e-4e6c-9ee9-6ef699256daf","name":"submitter-timeout-lgrnm-head"}
{"level":"info","ts":"2026-09-18T10:07:39.550Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"8f134714-ce30-4cff-ba0d-723c39698f89","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:39.550Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"8f134714-ce30-4cff-ba0d-723c39698f89","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:39.550Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"8f134714-ce30-4cff-ba0d-723c39698f89","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:39.550Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"8f134714-ce30-4cff-ba0d-723c39698f89"}
{"level":"info","ts":"2026-09-18T10:07:39.550Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"8f134714-ce30-4cff-ba0d-723c39698f89","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:39.550Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"8f134714-ce30-4cff-ba0d-723c39698f89","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:39.562Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"92f1493e-c46e-4e6c-9ee9-6ef699256daf","name":"submitter-timeout-lgrnm-workers"}
{"level":"info","ts":"2026-09-18T10:07:39.562Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"92f1493e-c46e-4e6c-9ee9-6ef699256daf","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:40.256Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"602085a0-9b4e-489d-8e74-1f95c96bf30c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:40.256Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"602085a0-9b4e-489d-8e74-1f95c96bf30c","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:40.256Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"602085a0-9b4e-489d-8e74-1f95c96bf30c","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:40.256Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"602085a0-9b4e-489d-8e74-1f95c96bf30c"}
{"level":"info","ts":"2026-09-18T10:07:40.256Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"602085a0-9b4e-489d-8e74-1f95c96bf30c","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:40.256Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"602085a0-9b4e-489d-8e74-1f95c96bf30c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:41.531Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"c24cfd5e-5075-4068-a087-1b1d59f9c7a4"}
{"level":"info","ts":"2026-09-18T10:07:41.531Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"c24cfd5e-5075-4068-a087-1b1d59f9c7a4"}
{"level":"info","ts":"2026-09-18T10:07:41.531Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"c24cfd5e-5075-4068-a087-1b1d59f9c7a4"}
{"level":"info","ts":"2026-09-18T10:07:41.531Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"c24cfd5e-5075-4068-a087-1b1d59f9c7a4","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:07:41.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"c24cfd5e-5075-4068-a087-1b1d59f9c7a4","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:07:41.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"c24cfd5e-5075-4068-a087-1b1d59f9c7a4","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:41.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"c24cfd5e-5075-4068-a087-1b1d59f9c7a4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:07:41.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"c24cfd5e-5075-4068-a087-1b1d59f9c7a4","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:41.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"c24cfd5e-5075-4068-a087-1b1d59f9c7a4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:41.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"c24cfd5e-5075-4068-a087-1b1d59f9c7a4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:07:41.532Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"c24cfd5e-5075-4068-a087-1b1d59f9c7a4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:07:41.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"c24cfd5e-5075-4068-a087-1b1d59f9c7a4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:41.532Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"c24cfd5e-5075-4068-a087-1b1d59f9c7a4","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:06:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:07:41.532Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"c24cfd5e-5075-4068-a087-1b1d59f9c7a4","podName":"submitter-timeout-lgrnm-head-8n6xl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:06:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:07:41.532Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"c24cfd5e-5075-4068-a087-1b1d59f9c7a4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:07:41.532Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"c24cfd5e-5075-4068-a087-1b1d59f9c7a4","seconds":300}
{"level":"info","ts":"2026-09-18T10:07:43.257Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c6f1bad2-5186-406a-960e-423660c977d5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:43.257Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c6f1bad2-5186-406a-960e-423660c977d5","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:43.257Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c6f1bad2-5186-406a-960e-423660c977d5","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:43.257Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c6f1bad2-5186-406a-960e-423660c977d5"}
{"level":"info","ts":"2026-09-18T10:07:43.257Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c6f1bad2-5186-406a-960e-423660c977d5","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:43.257Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c6f1bad2-5186-406a-960e-423660c977d5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:44.771Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"f5677e44-61ff-47b7-8226-3f14617d1ce7"}
{"level":"info","ts":"2026-09-18T10:07:44.771Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"f5677e44-61ff-47b7-8226-3f14617d1ce7"}
{"level":"info","ts":"2026-09-18T10:07:44.771Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"f5677e44-61ff-47b7-8226-3f14617d1ce7"}
{"level":"info","ts":"2026-09-18T10:07:44.771Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"f5677e44-61ff-47b7-8226-3f14617d1ce7","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:07:44.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"f5677e44-61ff-47b7-8226-3f14617d1ce7","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:07:44.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"f5677e44-61ff-47b7-8226-3f14617d1ce7","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:44.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"f5677e44-61ff-47b7-8226-3f14617d1ce7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:07:44.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"f5677e44-61ff-47b7-8226-3f14617d1ce7","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:44.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"f5677e44-61ff-47b7-8226-3f14617d1ce7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:44.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"f5677e44-61ff-47b7-8226-3f14617d1ce7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:07:44.771Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"f5677e44-61ff-47b7-8226-3f14617d1ce7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:07:44.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"f5677e44-61ff-47b7-8226-3f14617d1ce7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:44.771Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"f5677e44-61ff-47b7-8226-3f14617d1ce7","podName":"submitter-timeout-lgrnm-small-group-worker-xcz2v","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:06:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:07:44.771Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"f5677e44-61ff-47b7-8226-3f14617d1ce7","podName":"submitter-timeout-lgrnm-small-group-worker-xcz2v","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:06:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:07:44.771Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"f5677e44-61ff-47b7-8226-3f14617d1ce7","name":"submitter-timeout-lgrnm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:07:44.794Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6ed2cffc-9747-4ce0-9512-95580b1726f4","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:44.794Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"d4c32f41-1732-4ebd-93f7-b1acb25cc836","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:44.794Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6ed2cffc-9747-4ce0-9512-95580b1726f4","pod":"submitter-timeout-lgrnm-head-8n6xl","ip":"10.133.0.45"}
{"level":"info","ts":"2026-09-18T10:07:44.794Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6ed2cffc-9747-4ce0-9512-95580b1726f4","pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","ip":"10.132.0.52"}
{"level":"info","ts":"2026-09-18T10:07:44.794Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6ed2cffc-9747-4ce0-9512-95580b1726f4","rayCluster":"submitter-timeout-lgrnm","podIPs":["10.133.0.45","10.132.0.52"]}
{"level":"info","ts":"2026-09-18T10:07:44.794Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"165b2258-6453-4e8c-a3eb-33a2906383e7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:44.794Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"165b2258-6453-4e8c-a3eb-33a2906383e7","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:44.794Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"165b2258-6453-4e8c-a3eb-33a2906383e7","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:44.794Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"165b2258-6453-4e8c-a3eb-33a2906383e7"}
{"level":"info","ts":"2026-09-18T10:07:44.794Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"165b2258-6453-4e8c-a3eb-33a2906383e7","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:44.794Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"165b2258-6453-4e8c-a3eb-33a2906383e7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:44.794Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"165b2258-6453-4e8c-a3eb-33a2906383e7","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:07:44.818Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fd192582-0463-4171-ac4e-1ea3cdc1e36d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:44.818Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fd192582-0463-4171-ac4e-1ea3cdc1e36d","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:44.818Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fd192582-0463-4171-ac4e-1ea3cdc1e36d","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:44.818Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fd192582-0463-4171-ac4e-1ea3cdc1e36d"}
{"level":"info","ts":"2026-09-18T10:07:44.818Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fd192582-0463-4171-ac4e-1ea3cdc1e36d","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:44.818Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"fd192582-0463-4171-ac4e-1ea3cdc1e36d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:44.826Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"d4c32f41-1732-4ebd-93f7-b1acb25cc836","name":"submitter-timeout-lgrnm-head"}
{"level":"info","ts":"2026-09-18T10:07:44.850Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"d4c32f41-1732-4ebd-93f7-b1acb25cc836","name":"submitter-timeout-lgrnm-workers"}
{"level":"info","ts":"2026-09-18T10:07:44.850Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"d4c32f41-1732-4ebd-93f7-b1acb25cc836","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:45.581Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"1349fd40-c3f7-45f6-b4e1-9d1a3ac73211","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:07:45.581Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"1349fd40-c3f7-45f6-b4e1-9d1a3ac73211","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:07:45.581Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"1349fd40-c3f7-45f6-b4e1-9d1a3ac73211","pod":"long-running-5qfl8-small-group-worker-tbqtj","ip":"10.132.0.47"}
{"level":"info","ts":"2026-09-18T10:07:45.581Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"1349fd40-c3f7-45f6-b4e1-9d1a3ac73211","rayCluster":"long-running-5qfl8","podIPs":["10.133.0.42","10.132.0.47"]}
{"level":"info","ts":"2026-09-18T10:07:46.257Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"09542d3f-2dd0-4912-ad54-39f2797a0dc6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:46.258Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"09542d3f-2dd0-4912-ad54-39f2797a0dc6","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:46.258Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"09542d3f-2dd0-4912-ad54-39f2797a0dc6","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:46.258Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"09542d3f-2dd0-4912-ad54-39f2797a0dc6"}
{"level":"info","ts":"2026-09-18T10:07:46.258Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"09542d3f-2dd0-4912-ad54-39f2797a0dc6","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:46.258Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"09542d3f-2dd0-4912-ad54-39f2797a0dc6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:46.798Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"6181db65-027a-421f-86cc-d74f9e45ec81"}
{"level":"info","ts":"2026-09-18T10:07:46.798Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"6181db65-027a-421f-86cc-d74f9e45ec81"}
{"level":"info","ts":"2026-09-18T10:07:46.798Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"6181db65-027a-421f-86cc-d74f9e45ec81"}
{"level":"info","ts":"2026-09-18T10:07:46.798Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"6181db65-027a-421f-86cc-d74f9e45ec81","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:07:46.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"6181db65-027a-421f-86cc-d74f9e45ec81","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:07:46.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"6181db65-027a-421f-86cc-d74f9e45ec81","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:46.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"6181db65-027a-421f-86cc-d74f9e45ec81","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:07:46.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"6181db65-027a-421f-86cc-d74f9e45ec81","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:46.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"6181db65-027a-421f-86cc-d74f9e45ec81","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:46.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"6181db65-027a-421f-86cc-d74f9e45ec81","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:07:46.798Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"6181db65-027a-421f-86cc-d74f9e45ec81","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:07:46.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"6181db65-027a-421f-86cc-d74f9e45ec81","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:46.799Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"6181db65-027a-421f-86cc-d74f9e45ec81","podName":"submitter-timeout-lgrnm-small-group-worker-xcz2v","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:06:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:07:46.799Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"6181db65-027a-421f-86cc-d74f9e45ec81","podName":"submitter-timeout-lgrnm-small-group-worker-xcz2v","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:06:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:07:46.799Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"6181db65-027a-421f-86cc-d74f9e45ec81","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:07:46.799Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"6181db65-027a-421f-86cc-d74f9e45ec81","seconds":300}
{"level":"info","ts":"2026-09-18T10:07:49.258Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"914cc44c-d4e1-4ee2-a4d0-337526000779","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:49.258Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"914cc44c-d4e1-4ee2-a4d0-337526000779","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:49.258Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"914cc44c-d4e1-4ee2-a4d0-337526000779","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:49.258Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"914cc44c-d4e1-4ee2-a4d0-337526000779"}
{"level":"info","ts":"2026-09-18T10:07:49.258Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"914cc44c-d4e1-4ee2-a4d0-337526000779","RayCluster":"submitter-timeout-lgrnm","State":""}
{"level":"info","ts":"2026-09-18T10:07:49.258Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"914cc44c-d4e1-4ee2-a4d0-337526000779","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:50.537Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ff733011-0887-4ebd-bbb8-6a3577f7ca88"}
{"level":"info","ts":"2026-09-18T10:07:50.537Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ff733011-0887-4ebd-bbb8-6a3577f7ca88"}
{"level":"info","ts":"2026-09-18T10:07:50.537Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ff733011-0887-4ebd-bbb8-6a3577f7ca88"}
{"level":"info","ts":"2026-09-18T10:07:50.537Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ff733011-0887-4ebd-bbb8-6a3577f7ca88","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:07:50.537Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ff733011-0887-4ebd-bbb8-6a3577f7ca88","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:07:50.537Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ff733011-0887-4ebd-bbb8-6a3577f7ca88","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:50.537Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ff733011-0887-4ebd-bbb8-6a3577f7ca88","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:07:50.537Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ff733011-0887-4ebd-bbb8-6a3577f7ca88","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:50.537Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ff733011-0887-4ebd-bbb8-6a3577f7ca88","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:50.537Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ff733011-0887-4ebd-bbb8-6a3577f7ca88","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:07:50.537Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ff733011-0887-4ebd-bbb8-6a3577f7ca88","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:07:50.537Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ff733011-0887-4ebd-bbb8-6a3577f7ca88","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:50.538Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"ff733011-0887-4ebd-bbb8-6a3577f7ca88","name":"submitter-timeout-lgrnm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:07:50.554Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"59593805-183c-4efb-93d5-63ee3645a9b6","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:50.554Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d54bf010-18a4-4f3d-b69b-f4b287473bf9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:50.554Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d54bf010-18a4-4f3d-b69b-f4b287473bf9","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:50.554Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"59593805-183c-4efb-93d5-63ee3645a9b6","pod":"submitter-timeout-lgrnm-head-8n6xl","ip":"10.133.0.45"}
{"level":"info","ts":"2026-09-18T10:07:50.554Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"59593805-183c-4efb-93d5-63ee3645a9b6","pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","ip":"10.132.0.52"}
{"level":"info","ts":"2026-09-18T10:07:50.554Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d54bf010-18a4-4f3d-b69b-f4b287473bf9","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:50.554Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"59593805-183c-4efb-93d5-63ee3645a9b6","rayCluster":"submitter-timeout-lgrnm","podIPs":["10.133.0.45","10.132.0.52"]}
{"level":"info","ts":"2026-09-18T10:07:50.554Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d54bf010-18a4-4f3d-b69b-f4b287473bf9"}
{"level":"info","ts":"2026-09-18T10:07:50.554Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"f1ef373d-b3c3-4574-9928-30064c27705e","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:50.568Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d54bf010-18a4-4f3d-b69b-f4b287473bf9","Kubernetes Job":"submitter-timeout"}
{"level":"info","ts":"2026-09-18T10:07:50.568Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d54bf010-18a4-4f3d-b69b-f4b287473bf9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:50.568Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"d54bf010-18a4-4f3d-b69b-f4b287473bf9","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:07:50.577Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"f1ef373d-b3c3-4574-9928-30064c27705e","name":"submitter-timeout-lgrnm-head"}
{"level":"info","ts":"2026-09-18T10:07:50.588Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0b03ece2-cc11-4fe4-b635-b0d57ebe455c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:50.588Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0b03ece2-cc11-4fe4-b635-b0d57ebe455c","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:50.588Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0b03ece2-cc11-4fe4-b635-b0d57ebe455c","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:50.588Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0b03ece2-cc11-4fe4-b635-b0d57ebe455c"}
{"level":"info","ts":"2026-09-18T10:07:50.588Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0b03ece2-cc11-4fe4-b635-b0d57ebe455c","Kubernetes Job":"submitter-timeout"}
{"level":"info","ts":"2026-09-18T10:07:50.588Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0b03ece2-cc11-4fe4-b635-b0d57ebe455c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:50.588Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0b03ece2-cc11-4fe4-b635-b0d57ebe455c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:07:50.597Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"f1ef373d-b3c3-4574-9928-30064c27705e","name":"submitter-timeout-lgrnm-workers"}
{"level":"info","ts":"2026-09-18T10:07:50.597Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"f1ef373d-b3c3-4574-9928-30064c27705e","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:50.599Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0b03ece2-cc11-4fe4-b635-b0d57ebe455c","error":"Operation cannot be fulfilled on rayjobs.ray.io \"submitter-timeout\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T10:07:50.599Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0b03ece2-cc11-4fe4-b635-b0d57ebe455c"}
{"level":"error","ts":"2026-09-18T10:07:50.599Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0b03ece2-cc11-4fe4-b635-b0d57ebe455c","error":"Operation cannot be fulfilled on rayjobs.ray.io \"submitter-timeout\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:50.599Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"44071092-ea6d-4466-91bf-82ca3a95d79c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:50.599Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"44071092-ea6d-4466-91bf-82ca3a95d79c","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:50.599Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"44071092-ea6d-4466-91bf-82ca3a95d79c","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:50.600Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"44071092-ea6d-4466-91bf-82ca3a95d79c"}
{"level":"info","ts":"2026-09-18T10:07:50.600Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"44071092-ea6d-4466-91bf-82ca3a95d79c","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:07:50.617Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"44071092-ea6d-4466-91bf-82ca3a95d79c","JobId":"submitter-timeout-g27h7","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:50.617Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"44071092-ea6d-4466-91bf-82ca3a95d79c"}
{"level":"error","ts":"2026-09-18T10:07:50.617Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"44071092-ea6d-4466-91bf-82ca3a95d79c","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:50.617Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"224835f3-e5b5-4cd1-8ee4-2fe29e6bcaac","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:50.617Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"224835f3-e5b5-4cd1-8ee4-2fe29e6bcaac","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:50.617Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"224835f3-e5b5-4cd1-8ee4-2fe29e6bcaac","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:50.617Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"224835f3-e5b5-4cd1-8ee4-2fe29e6bcaac"}
{"level":"info","ts":"2026-09-18T10:07:50.617Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"224835f3-e5b5-4cd1-8ee4-2fe29e6bcaac","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:07:50.621Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"224835f3-e5b5-4cd1-8ee4-2fe29e6bcaac","JobId":"submitter-timeout-g27h7","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:50.621Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"224835f3-e5b5-4cd1-8ee4-2fe29e6bcaac"}
{"level":"error","ts":"2026-09-18T10:07:50.621Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"224835f3-e5b5-4cd1-8ee4-2fe29e6bcaac","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:50.628Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"edd63436-2c01-4804-b3ff-58367d9352da","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:50.628Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"edd63436-2c01-4804-b3ff-58367d9352da","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:50.628Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"edd63436-2c01-4804-b3ff-58367d9352da","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:50.628Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"edd63436-2c01-4804-b3ff-58367d9352da"}
{"level":"info","ts":"2026-09-18T10:07:50.628Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"edd63436-2c01-4804-b3ff-58367d9352da","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:07:50.670Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"edd63436-2c01-4804-b3ff-58367d9352da","JobId":"submitter-timeout-g27h7","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:50.670Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"edd63436-2c01-4804-b3ff-58367d9352da"}
{"level":"error","ts":"2026-09-18T10:07:50.670Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"edd63436-2c01-4804-b3ff-58367d9352da","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:50.710Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"e5d9e670-35e6-47e8-9cb2-4e359503e555","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:50.710Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"e5d9e670-35e6-47e8-9cb2-4e359503e555","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:50.710Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"e5d9e670-35e6-47e8-9cb2-4e359503e555","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:50.710Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"e5d9e670-35e6-47e8-9cb2-4e359503e555"}
{"level":"info","ts":"2026-09-18T10:07:50.711Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"e5d9e670-35e6-47e8-9cb2-4e359503e555","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:07:50.715Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"e5d9e670-35e6-47e8-9cb2-4e359503e555","JobId":"submitter-timeout-g27h7","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:50.715Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"e5d9e670-35e6-47e8-9cb2-4e359503e555"}
{"level":"error","ts":"2026-09-18T10:07:50.715Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"e5d9e670-35e6-47e8-9cb2-4e359503e555","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:50.795Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c50ded3e-ab94-4529-8261-94753c180b22","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:50.795Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c50ded3e-ab94-4529-8261-94753c180b22","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:50.795Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c50ded3e-ab94-4529-8261-94753c180b22","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:50.796Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c50ded3e-ab94-4529-8261-94753c180b22"}
{"level":"info","ts":"2026-09-18T10:07:50.796Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c50ded3e-ab94-4529-8261-94753c180b22","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:07:50.800Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c50ded3e-ab94-4529-8261-94753c180b22","JobId":"submitter-timeout-g27h7","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:50.800Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c50ded3e-ab94-4529-8261-94753c180b22"}
{"level":"error","ts":"2026-09-18T10:07:50.800Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c50ded3e-ab94-4529-8261-94753c180b22","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:50.961Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"13119b2e-829c-4fa6-96d1-d5ad58ec87ea","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:50.961Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"13119b2e-829c-4fa6-96d1-d5ad58ec87ea","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:50.961Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"13119b2e-829c-4fa6-96d1-d5ad58ec87ea","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:50.961Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"13119b2e-829c-4fa6-96d1-d5ad58ec87ea"}
{"level":"info","ts":"2026-09-18T10:07:50.962Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"13119b2e-829c-4fa6-96d1-d5ad58ec87ea","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:07:50.967Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"13119b2e-829c-4fa6-96d1-d5ad58ec87ea","JobId":"submitter-timeout-g27h7","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:50.967Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"13119b2e-829c-4fa6-96d1-d5ad58ec87ea"}
{"level":"error","ts":"2026-09-18T10:07:50.967Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"13119b2e-829c-4fa6-96d1-d5ad58ec87ea","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:51.287Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"4025cca6-eb3d-4a2f-b8db-86e872bfc949","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:51.287Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"4025cca6-eb3d-4a2f-b8db-86e872bfc949","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:51.287Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"4025cca6-eb3d-4a2f-b8db-86e872bfc949","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:51.287Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"4025cca6-eb3d-4a2f-b8db-86e872bfc949"}
{"level":"info","ts":"2026-09-18T10:07:51.287Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"4025cca6-eb3d-4a2f-b8db-86e872bfc949","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:07:51.293Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"4025cca6-eb3d-4a2f-b8db-86e872bfc949","JobId":"submitter-timeout-g27h7","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:51.293Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"4025cca6-eb3d-4a2f-b8db-86e872bfc949"}
{"level":"error","ts":"2026-09-18T10:07:51.293Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"4025cca6-eb3d-4a2f-b8db-86e872bfc949","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:51.605Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5e85a71f-2c37-4750-85cd-2fbd2e04318d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:51.605Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5e85a71f-2c37-4750-85cd-2fbd2e04318d","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:51.605Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5e85a71f-2c37-4750-85cd-2fbd2e04318d","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:51.605Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5e85a71f-2c37-4750-85cd-2fbd2e04318d"}
{"level":"info","ts":"2026-09-18T10:07:51.605Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5e85a71f-2c37-4750-85cd-2fbd2e04318d","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:07:51.610Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5e85a71f-2c37-4750-85cd-2fbd2e04318d","JobId":"submitter-timeout-g27h7","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:51.610Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5e85a71f-2c37-4750-85cd-2fbd2e04318d"}
{"level":"error","ts":"2026-09-18T10:07:51.610Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5e85a71f-2c37-4750-85cd-2fbd2e04318d","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:51.934Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"a8528096-2fc2-4b53-9ad9-3495553d2325","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:51.934Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"a8528096-2fc2-4b53-9ad9-3495553d2325","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:51.934Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"a8528096-2fc2-4b53-9ad9-3495553d2325","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:51.934Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"a8528096-2fc2-4b53-9ad9-3495553d2325"}
{"level":"info","ts":"2026-09-18T10:07:51.934Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"a8528096-2fc2-4b53-9ad9-3495553d2325","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:07:51.939Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"a8528096-2fc2-4b53-9ad9-3495553d2325","JobId":"submitter-timeout-g27h7","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:51.939Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"a8528096-2fc2-4b53-9ad9-3495553d2325"}
{"level":"error","ts":"2026-09-18T10:07:51.939Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"a8528096-2fc2-4b53-9ad9-3495553d2325","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:52.555Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0213f55f-65ef-4de7-a5cf-f83206e8334f"}
{"level":"info","ts":"2026-09-18T10:07:52.555Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0213f55f-65ef-4de7-a5cf-f83206e8334f"}
{"level":"info","ts":"2026-09-18T10:07:52.555Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0213f55f-65ef-4de7-a5cf-f83206e8334f"}
{"level":"info","ts":"2026-09-18T10:07:52.555Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0213f55f-65ef-4de7-a5cf-f83206e8334f","1 head service found":"submitter-timeout-lgrnm-head-svc"}
{"level":"info","ts":"2026-09-18T10:07:52.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0213f55f-65ef-4de7-a5cf-f83206e8334f","Found 1 head Pod":"submitter-timeout-lgrnm-head-8n6xl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:07:52.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0213f55f-65ef-4de7-a5cf-f83206e8334f","head Pod":"submitter-timeout-lgrnm-head-8n6xl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-lgrnm-head-8n6xl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:52.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0213f55f-65ef-4de7-a5cf-f83206e8334f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:07:52.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0213f55f-65ef-4de7-a5cf-f83206e8334f","worker Pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-lgrnm-small-group-worker-xcz2v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:07:52.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0213f55f-65ef-4de7-a5cf-f83206e8334f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:52.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0213f55f-65ef-4de7-a5cf-f83206e8334f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:07:52.555Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0213f55f-65ef-4de7-a5cf-f83206e8334f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:07:52.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0213f55f-65ef-4de7-a5cf-f83206e8334f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:07:52.556Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0213f55f-65ef-4de7-a5cf-f83206e8334f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:07:52.556Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"0213f55f-65ef-4de7-a5cf-f83206e8334f","seconds":300}
{"level":"info","ts":"2026-09-18T10:07:54.500Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"3c87ebe1-cc04-44d7-ad3f-4a82d91f1b38","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:54.500Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"3c87ebe1-cc04-44d7-ad3f-4a82d91f1b38","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:54.500Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"3c87ebe1-cc04-44d7-ad3f-4a82d91f1b38","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:54.500Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"3c87ebe1-cc04-44d7-ad3f-4a82d91f1b38"}
{"level":"info","ts":"2026-09-18T10:07:54.500Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"3c87ebe1-cc04-44d7-ad3f-4a82d91f1b38","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:07:54.505Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"3c87ebe1-cc04-44d7-ad3f-4a82d91f1b38","JobId":"submitter-timeout-g27h7","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:54.505Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"3c87ebe1-cc04-44d7-ad3f-4a82d91f1b38"}
{"level":"error","ts":"2026-09-18T10:07:54.505Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"3c87ebe1-cc04-44d7-ad3f-4a82d91f1b38","error":"Job submitter-timeout-g27h7 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:07:59.625Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f096b3b0-fd42-4efe-8b0e-bb5ca3c64204","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:59.625Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f096b3b0-fd42-4efe-8b0e-bb5ca3c64204","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:59.625Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f096b3b0-fd42-4efe-8b0e-bb5ca3c64204","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:59.625Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f096b3b0-fd42-4efe-8b0e-bb5ca3c64204"}
{"level":"info","ts":"2026-09-18T10:07:59.625Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f096b3b0-fd42-4efe-8b0e-bb5ca3c64204","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:07:59.632Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f096b3b0-fd42-4efe-8b0e-bb5ca3c64204","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:07:59.632Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f096b3b0-fd42-4efe-8b0e-bb5ca3c64204","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:07:59.651Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"29a60720-905e-4c3a-92fe-ef7878540c81","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:07:59.651Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"29a60720-905e-4c3a-92fe-ef7878540c81","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:07:59.651Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"29a60720-905e-4c3a-92fe-ef7878540c81","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:07:59.651Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"29a60720-905e-4c3a-92fe-ef7878540c81"}
{"level":"info","ts":"2026-09-18T10:07:59.651Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"29a60720-905e-4c3a-92fe-ef7878540c81","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:07:59.654Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"29a60720-905e-4c3a-92fe-ef7878540c81","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:02.652Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"feb5ba1b-c93e-42ae-af3b-95775b7def12","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:02.652Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"feb5ba1b-c93e-42ae-af3b-95775b7def12","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:02.652Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"feb5ba1b-c93e-42ae-af3b-95775b7def12","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:02.652Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"feb5ba1b-c93e-42ae-af3b-95775b7def12"}
{"level":"info","ts":"2026-09-18T10:08:02.652Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"feb5ba1b-c93e-42ae-af3b-95775b7def12","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:08:02.655Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"feb5ba1b-c93e-42ae-af3b-95775b7def12","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:02.655Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"feb5ba1b-c93e-42ae-af3b-95775b7def12","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:08:02.680Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"8fef3a66-3084-4044-9728-4662d1b420a0","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:02.680Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"8fef3a66-3084-4044-9728-4662d1b420a0","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:02.680Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"8fef3a66-3084-4044-9728-4662d1b420a0","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:02.680Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"8fef3a66-3084-4044-9728-4662d1b420a0"}
{"level":"info","ts":"2026-09-18T10:08:02.680Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"8fef3a66-3084-4044-9728-4662d1b420a0","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:08:02.682Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"8fef3a66-3084-4044-9728-4662d1b420a0","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:03.167Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"ef6ae335-09f3-48ec-b994-b29d6c120ea6","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:03.167Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"ef6ae335-09f3-48ec-b994-b29d6c120ea6","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:03.167Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"ef6ae335-09f3-48ec-b994-b29d6c120ea6","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:03.167Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"ef6ae335-09f3-48ec-b994-b29d6c120ea6"}
{"level":"info","ts":"2026-09-18T10:08:03.167Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"ef6ae335-09f3-48ec-b994-b29d6c120ea6","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:08:03.170Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"ef6ae335-09f3-48ec-b994-b29d6c120ea6","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:05.680Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f9c781a2-b7a3-4b70-8176-53964544570d","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:05.680Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f9c781a2-b7a3-4b70-8176-53964544570d","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:05.680Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f9c781a2-b7a3-4b70-8176-53964544570d","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:05.680Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f9c781a2-b7a3-4b70-8176-53964544570d"}
{"level":"info","ts":"2026-09-18T10:08:05.680Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f9c781a2-b7a3-4b70-8176-53964544570d","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:08:05.683Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f9c781a2-b7a3-4b70-8176-53964544570d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:08.684Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"87b55c3e-6f4e-4b98-bf9a-d5f947b67bda","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:08.684Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"87b55c3e-6f4e-4b98-bf9a-d5f947b67bda","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:08.684Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"87b55c3e-6f4e-4b98-bf9a-d5f947b67bda","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:08.684Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"87b55c3e-6f4e-4b98-bf9a-d5f947b67bda"}
{"level":"info","ts":"2026-09-18T10:08:08.684Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"87b55c3e-6f4e-4b98-bf9a-d5f947b67bda","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:08:08.687Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"87b55c3e-6f4e-4b98-bf9a-d5f947b67bda","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:11.687Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f484f3c8-76e4-42da-8e6d-10979d0dd0f4","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:11.687Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f484f3c8-76e4-42da-8e6d-10979d0dd0f4","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:11.687Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f484f3c8-76e4-42da-8e6d-10979d0dd0f4","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:11.688Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f484f3c8-76e4-42da-8e6d-10979d0dd0f4"}
{"level":"info","ts":"2026-09-18T10:08:11.688Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f484f3c8-76e4-42da-8e6d-10979d0dd0f4","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:08:11.690Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f484f3c8-76e4-42da-8e6d-10979d0dd0f4","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:14.691Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f87852c4-7da0-415d-b8b6-d4287ebeb43a","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:14.691Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f87852c4-7da0-415d-b8b6-d4287ebeb43a","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:14.691Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f87852c4-7da0-415d-b8b6-d4287ebeb43a","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:14.691Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f87852c4-7da0-415d-b8b6-d4287ebeb43a"}
{"level":"info","ts":"2026-09-18T10:08:14.691Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f87852c4-7da0-415d-b8b6-d4287ebeb43a","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:08:14.694Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"f87852c4-7da0-415d-b8b6-d4287ebeb43a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:17.695Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"2f7824f0-2de2-4d7e-8341-4a4b1ba694e9","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:17.695Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"2f7824f0-2de2-4d7e-8341-4a4b1ba694e9","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:17.695Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"2f7824f0-2de2-4d7e-8341-4a4b1ba694e9","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:17.695Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"2f7824f0-2de2-4d7e-8341-4a4b1ba694e9"}
{"level":"info","ts":"2026-09-18T10:08:17.695Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"2f7824f0-2de2-4d7e-8341-4a4b1ba694e9","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:08:17.698Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"2f7824f0-2de2-4d7e-8341-4a4b1ba694e9","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:20.699Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c1ceaff7-e88a-4c14-b6ea-1ae37b2c0715","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:20.699Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c1ceaff7-e88a-4c14-b6ea-1ae37b2c0715","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:20.699Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c1ceaff7-e88a-4c14-b6ea-1ae37b2c0715","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:20.699Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c1ceaff7-e88a-4c14-b6ea-1ae37b2c0715"}
{"level":"info","ts":"2026-09-18T10:08:20.699Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c1ceaff7-e88a-4c14-b6ea-1ae37b2c0715","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:08:20.702Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"c1ceaff7-e88a-4c14-b6ea-1ae37b2c0715","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:23.703Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"97a2ac5b-fb36-4c12-9576-daa69fb91f1f","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:23.703Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"97a2ac5b-fb36-4c12-9576-daa69fb91f1f","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:23.703Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"97a2ac5b-fb36-4c12-9576-daa69fb91f1f","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:23.703Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"97a2ac5b-fb36-4c12-9576-daa69fb91f1f"}
{"level":"info","ts":"2026-09-18T10:08:23.703Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"97a2ac5b-fb36-4c12-9576-daa69fb91f1f","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:08:23.706Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"97a2ac5b-fb36-4c12-9576-daa69fb91f1f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:26.707Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5d6e24bb-209e-4d6f-9dff-6b54ff5ee161","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:26.707Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5d6e24bb-209e-4d6f-9dff-6b54ff5ee161","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:26.707Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5d6e24bb-209e-4d6f-9dff-6b54ff5ee161","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:26.707Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5d6e24bb-209e-4d6f-9dff-6b54ff5ee161"}
{"level":"info","ts":"2026-09-18T10:08:26.707Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5d6e24bb-209e-4d6f-9dff-6b54ff5ee161","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:08:26.710Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"5d6e24bb-209e-4d6f-9dff-6b54ff5ee161","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:29.711Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"38683af2-080f-4989-b419-339e4003deaf","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:29.711Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"38683af2-080f-4989-b419-339e4003deaf","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:29.711Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"38683af2-080f-4989-b419-339e4003deaf","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:29.711Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"38683af2-080f-4989-b419-339e4003deaf"}
{"level":"info","ts":"2026-09-18T10:08:29.711Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"38683af2-080f-4989-b419-339e4003deaf","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:08:29.714Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"38683af2-080f-4989-b419-339e4003deaf","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:32.715Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"efc3b41b-35cd-4752-8487-de3eb37b4aab","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:32.715Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"efc3b41b-35cd-4752-8487-de3eb37b4aab","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:32.715Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"efc3b41b-35cd-4752-8487-de3eb37b4aab","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:32.715Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"efc3b41b-35cd-4752-8487-de3eb37b4aab"}
{"level":"info","ts":"2026-09-18T10:08:32.715Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"efc3b41b-35cd-4752-8487-de3eb37b4aab","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:08:32.718Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"efc3b41b-35cd-4752-8487-de3eb37b4aab","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:34.670Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"f9a5eddf-d6fd-4159-9770-40b2b4ed8f69","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:34.670Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"f9a5eddf-d6fd-4159-9770-40b2b4ed8f69","pod":"submitter-timeout-lgrnm-head-8n6xl","ip":"10.133.0.45"}
{"level":"info","ts":"2026-09-18T10:08:34.670Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"f9a5eddf-d6fd-4159-9770-40b2b4ed8f69","pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","ip":"10.132.0.52"}
{"level":"info","ts":"2026-09-18T10:08:34.670Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"f9a5eddf-d6fd-4159-9770-40b2b4ed8f69","rayCluster":"submitter-timeout-lgrnm","podIPs":["10.133.0.45","10.132.0.52"]}
{"level":"info","ts":"2026-09-18T10:08:35.719Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0d26ee31-2014-4dc1-bb0c-1bbd6066705b","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:35.719Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0d26ee31-2014-4dc1-bb0c-1bbd6066705b","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:35.719Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0d26ee31-2014-4dc1-bb0c-1bbd6066705b","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:35.719Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0d26ee31-2014-4dc1-bb0c-1bbd6066705b"}
{"level":"info","ts":"2026-09-18T10:08:35.719Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0d26ee31-2014-4dc1-bb0c-1bbd6066705b","rayCluster":"submitter-timeout-lgrnm","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:08:35.719Z","logger":"controllers.RayJob","msg":"The RayJob has passed the submitterFinishedTimeoutSeconds. Transition the status to terminal.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0d26ee31-2014-4dc1-bb0c-1bbd6066705b","SubmitterFinishedTime":"2026-09-18T10:08:03.000Z","SubmitterFinishedTimeoutSeconds":"30s"}
{"level":"info","ts":"2026-09-18T10:08:35.719Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0d26ee31-2014-4dc1-bb0c-1bbd6066705b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:07:58Z"},"jobId":"submitter-timeout-g27h7","rayClusterName":"submitter-timeout-lgrnm","dashboardURL":"submitter-timeout-lgrnm-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"JobDeploymentStatusTransitionGracePeriodExceeded","message":"The RayJob submitter finished at 2026-09-18 10:08:03 but the ray job did not reach terminal state within 30s","startTime":"2026-09-18T10:06:04Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:07:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:07:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.45","serviceIP":"10.133.0.45","podName":"submitter-timeout-lgrnm-head-8n6xl","serviceName":"submitter-timeout-lgrnm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:07:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:07:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:06:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:06:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:35.719Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0d26ee31-2014-4dc1-bb0c-1bbd6066705b","old JobStatus":"RUNNING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-09-18T10:08:35.738Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"50527ea0-10be-44c3-b281-876859f20352","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:35.738Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"50527ea0-10be-44c3-b281-876859f20352","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:08:35 +0000 UTC","Now":"2026-09-18T10:08:35.738Z","ShutdownTime":"2026-09-18T10:08:35.000Z"}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"50527ea0-10be-44c3-b281-876859f20352","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"50527ea0-10be-44c3-b281-876859f20352","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"50527ea0-10be-44c3-b281-876859f20352","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"reconcileID":"e8225000-a5e4-4ae8-b0bd-66933ed11fe3"}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"0598edac-b523-4741-aa33-69f993f048c6"}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"d8b0e5f1-f750-4b99-889b-0eda17cb8ec8","rayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"27091048-e3eb-4d53-8a83-0dcad73d0df8","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"27091048-e3eb-4d53-8a83-0dcad73d0df8","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:08:35 +0000 UTC","Now":"2026-09-18T10:08:35.752Z","ShutdownTime":"2026-09-18T10:08:35.000Z"}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"27091048-e3eb-4d53-8a83-0dcad73d0df8","RayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"27091048-e3eb-4d53-8a83-0dcad73d0df8","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"27091048-e3eb-4d53-8a83-0dcad73d0df8","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6f206c66-3900-43e2-9a2b-f508c3f176a5","namespacedName":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6f206c66-3900-43e2-9a2b-f508c3f176a5","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6f206c66-3900-43e2-9a2b-f508c3f176a5","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"d8b0e5f1-f750-4b99-889b-0eda17cb8ec8","pod":"submitter-timeout-lgrnm-head-8n6xl","ip":"10.133.0.45"}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"d8b0e5f1-f750-4b99-889b-0eda17cb8ec8","pod":"submitter-timeout-lgrnm-small-group-worker-xcz2v","ip":"10.132.0.52"}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6f206c66-3900-43e2-9a2b-f508c3f176a5","namespace":"test-ns-ltx56","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6f206c66-3900-43e2-9a2b-f508c3f176a5","name":"kuberay-gateway-access","namespace":"test-ns-ltx56","remainingClusters":1}
{"level":"info","ts":"2026-09-18T10:08:35.752Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"d8b0e5f1-f750-4b99-889b-0eda17cb8ec8","rayCluster":"submitter-timeout-lgrnm","podIPs":["10.133.0.45","10.132.0.52"]}
{"level":"info","ts":"2026-09-18T10:08:35.764Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6f206c66-3900-43e2-9a2b-f508c3f176a5","name":"test-ns-ltx56-submitter-timeout-lgrnm","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T10:08:35.771Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6f206c66-3900-43e2-9a2b-f508c3f176a5","configMap":"kube-rbac-proxy-config-submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:35.777Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6f206c66-3900-43e2-9a2b-f508c3f176a5","serviceAccount":"submitter-timeout-lgrnm-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:08:35.801Z","logger":"raycluster-resource","msg":"default","name":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:35.802Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:08:35.812Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"6f206c66-3900-43e2-9a2b-f508c3f176a5","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:35.812Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"455743cd-dc4d-4da6-a4d0-abddf05b9761","namespacedName":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:35.812Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"455743cd-dc4d-4da6-a4d0-abddf05b9761","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:35.812Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"455743cd-dc4d-4da6-a4d0-abddf05b9761","cluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:35.812Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"455743cd-dc4d-4da6-a4d0-abddf05b9761","namespace":"test-ns-ltx56","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-09-18T10:08:35.812Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"455743cd-dc4d-4da6-a4d0-abddf05b9761","name":"kuberay-gateway-access","namespace":"test-ns-ltx56","remainingClusters":1}
{"level":"info","ts":"2026-09-18T10:08:35.813Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"b64b30e9-896d-4b76-855e-e3af87501162","namespace":"test-ns-ltx56","clusterName":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:35.813Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"449e03af-fb8a-4b95-9829-1a6de5b63a8d","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:35.813Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"449e03af-fb8a-4b95-9829-1a6de5b63a8d","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:08:35 +0000 UTC","Now":"2026-09-18T10:08:35.813Z","ShutdownTime":"2026-09-18T10:08:35.000Z"}
{"level":"info","ts":"2026-09-18T10:08:35.813Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"449e03af-fb8a-4b95-9829-1a6de5b63a8d","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:35.813Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"449e03af-fb8a-4b95-9829-1a6de5b63a8d","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T10:08:35.813Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"449e03af-fb8a-4b95-9829-1a6de5b63a8d","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:35.813Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"609936b5-220a-4fb3-9c0d-2cadd09cd899"}
{"level":"error","ts":"2026-09-18T10:08:35.817Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"455743cd-dc4d-4da6-a4d0-abddf05b9761","error":"rayclusters.ray.io \"submitter-timeout-lgrnm\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:196\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:137\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:35.817Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"455743cd-dc4d-4da6-a4d0-abddf05b9761"}
{"level":"error","ts":"2026-09-18T10:08:35.817Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"455743cd-dc4d-4da6-a4d0-abddf05b9761","error":"rayclusters.ray.io \"submitter-timeout-lgrnm\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:35.817Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"72f556b1-cfcd-4868-9fc2-7f29c81c9950","namespacedName":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:35.817Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"72f556b1-cfcd-4868-9fc2-7f29c81c9950"}
{"level":"info","ts":"2026-09-18T10:08:35.817Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"72f556b1-cfcd-4868-9fc2-7f29c81c9950","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:08:35.817Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"72f556b1-cfcd-4868-9fc2-7f29c81c9950","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:08:35.819Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"36b80338-2b4a-454e-9981-55fd027658f7"}
{"level":"info","ts":"2026-09-18T10:08:35.822Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"db330725-96c6-48d1-bdac-35f5ca5e8187","namespacedName":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:35.822Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"db330725-96c6-48d1-bdac-35f5ca5e8187"}
{"level":"info","ts":"2026-09-18T10:08:35.822Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"db330725-96c6-48d1-bdac-35f5ca5e8187","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:08:35.822Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"db330725-96c6-48d1-bdac-35f5ca5e8187","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:08:35.866Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"1c6b1050-5132-4f46-94dd-b3ee4b27bb90","namespacedName":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:35.866Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"1c6b1050-5132-4f46-94dd-b3ee4b27bb90"}
{"level":"info","ts":"2026-09-18T10:08:35.867Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"1c6b1050-5132-4f46-94dd-b3ee4b27bb90","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:08:35.867Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"1c6b1050-5132-4f46-94dd-b3ee4b27bb90","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:08:35.867Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"987a53d9-1668-48a9-92ac-ea0a7b4126c2"}
{"level":"info","ts":"2026-09-18T10:08:35.886Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"b64b30e9-896d-4b76-855e-e3af87501162","namespace":"test-ns-ltx56","clusterName":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:36.463Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a3852a88-9096-4f1f-a4f3-e18a6a528c92","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:36.463Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a3852a88-9096-4f1f-a4f3-e18a6a528c92","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T10:08:36.486Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a3852a88-9096-4f1f-a4f3-e18a6a528c92"}
{"level":"info","ts":"2026-09-18T10:08:36.486Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a3852a88-9096-4f1f-a4f3-e18a6a528c92","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T10:08:36.486Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a3852a88-9096-4f1f-a4f3-e18a6a528c92","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:08:36.486Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a3852a88-9096-4f1f-a4f3-e18a6a528c92","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:08:36.504Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"56a084bf-b39b-4540-8dfc-118a33e137eb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:36.504Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"56a084bf-b39b-4540-8dfc-118a33e137eb","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.504Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"56a084bf-b39b-4540-8dfc-118a33e137eb","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:36.504Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"56a084bf-b39b-4540-8dfc-118a33e137eb","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:36.509Z","logger":"raycluster-resource","msg":"default","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.509Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:08:36.530Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"56a084bf-b39b-4540-8dfc-118a33e137eb","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:36.530Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"56a084bf-b39b-4540-8dfc-118a33e137eb"}
{"level":"info","ts":"2026-09-18T10:08:36.530Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"56a084bf-b39b-4540-8dfc-118a33e137eb","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:36.530Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"56a084bf-b39b-4540-8dfc-118a33e137eb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:08:36.530Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"84b75d51-0519-46f5-9a3a-6c865d1cfcec","rayCluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.530Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"84b75d51-0519-46f5-9a3a-6c865d1cfcec","rayCluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.530Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"d48dc3f5-9af6-42a3-9be0-ecec52f5a5f6","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.530Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"54db3867-40b0-4ef5-9862-ef7b8f2196f6"}
{"level":"info","ts":"2026-09-18T10:08:36.530Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"54db3867-40b0-4ef5-9862-ef7b8f2196f6"}
{"level":"info","ts":"2026-09-18T10:08:36.530Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"54db3867-40b0-4ef5-9862-ef7b8f2196f6"}
{"level":"info","ts":"2026-09-18T10:08:36.530Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"56ae0fa9-9c99-4879-833e-8b93f02e980e","namespacedName":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:36.530Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"56ae0fa9-9c99-4879-833e-8b93f02e980e","mode":"IntegratedOAuth","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.530Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"56ae0fa9-9c99-4879-833e-8b93f02e980e","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.530Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"56ae0fa9-9c99-4879-833e-8b93f02e980e","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.531Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0059397a-b775-4748-a792-d50fc4599273","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:36.531Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0059397a-b775-4748-a792-d50fc4599273","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.531Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0059397a-b775-4748-a792-d50fc4599273","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:36.531Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0059397a-b775-4748-a792-d50fc4599273"}
{"level":"info","ts":"2026-09-18T10:08:36.531Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0059397a-b775-4748-a792-d50fc4599273","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:36.531Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0059397a-b775-4748-a792-d50fc4599273","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:08:36.537Z","logger":"raycluster-resource","msg":"default","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.537Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:08:36.547Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"52020e1f-eddf-431b-a7c8-5636330cfb79","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:36.547Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"52020e1f-eddf-431b-a7c8-5636330cfb79","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.547Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"52020e1f-eddf-431b-a7c8-5636330cfb79","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:36.547Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"52020e1f-eddf-431b-a7c8-5636330cfb79"}
{"level":"info","ts":"2026-09-18T10:08:36.547Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"52020e1f-eddf-431b-a7c8-5636330cfb79","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:36.547Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"52020e1f-eddf-431b-a7c8-5636330cfb79","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:08:36.550Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"d48dc3f5-9af6-42a3-9be0-ecec52f5a5f6","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.550Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"11ee93a9-2291-4642-a996-a27d09d8510c","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.554Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"56ae0fa9-9c99-4879-833e-8b93f02e980e","name":"cluster-status-update-m6crj-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:08:36.562Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"11ee93a9-2291-4642-a996-a27d09d8510c","name":"cluster-status-update-m6crj-head"}
{"level":"info","ts":"2026-09-18T10:08:36.576Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"56ae0fa9-9c99-4879-833e-8b93f02e980e","name":"test-ns-ltx56-cluster-status-update-m6crj","operation":"created"}
{"level":"info","ts":"2026-09-18T10:08:36.576Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"84b75d51-0519-46f5-9a3a-6c865d1cfcec","rayCluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.577Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"84b75d51-0519-46f5-9a3a-6c865d1cfcec","rayCluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.581Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"54db3867-40b0-4ef5-9862-ef7b8f2196f6","name":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:36.581Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"54db3867-40b0-4ef5-9862-ef7b8f2196f6","error":"head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"error","ts":"2026-09-18T10:08:36.581Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"54db3867-40b0-4ef5-9862-ef7b8f2196f6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:36.581Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"54db3867-40b0-4ef5-9862-ef7b8f2196f6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"info","ts":"2026-09-18T10:08:36.582Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"54db3867-40b0-4ef5-9862-ef7b8f2196f6","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.582Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"54db3867-40b0-4ef5-9862-ef7b8f2196f6","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.582Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"54db3867-40b0-4ef5-9862-ef7b8f2196f6","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.582Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"54db3867-40b0-4ef5-9862-ef7b8f2196f6","name":"cluster-status-update-m6crj","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:08:36.585Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"56ae0fa9-9c99-4879-833e-8b93f02e980e","name":"kube-rbac-proxy-config-cluster-status-update-m6crj","operation":"created"}
{"level":"info","ts":"2026-09-18T10:08:36.593Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"11ee93a9-2291-4642-a996-a27d09d8510c","name":"cluster-status-update-m6crj-workers"}
{"level":"info","ts":"2026-09-18T10:08:36.593Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"11ee93a9-2291-4642-a996-a27d09d8510c","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.597Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"54db3867-40b0-4ef5-9862-ef7b8f2196f6","name":"cluster-status-update-m6crj","error":"Operation cannot be fulfilled on rayclusters.ray.io \"cluster-status-update-m6crj\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj"}}
{"level":"info","ts":"2026-09-18T10:08:36.597Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"54db3867-40b0-4ef5-9862-ef7b8f2196f6"}
{"level":"error","ts":"2026-09-18T10:08:36.597Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"54db3867-40b0-4ef5-9862-ef7b8f2196f6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:36.598Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2b3ec8b6-519a-45f2-b8d6-be9c8ef105b2"}
{"level":"info","ts":"2026-09-18T10:08:36.598Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2b3ec8b6-519a-45f2-b8d6-be9c8ef105b2"}
{"level":"info","ts":"2026-09-18T10:08:36.598Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2b3ec8b6-519a-45f2-b8d6-be9c8ef105b2"}
{"level":"info","ts":"2026-09-18T10:08:36.598Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2b3ec8b6-519a-45f2-b8d6-be9c8ef105b2","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:36.598Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2b3ec8b6-519a-45f2-b8d6-be9c8ef105b2","error":"head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"error","ts":"2026-09-18T10:08:36.598Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2b3ec8b6-519a-45f2-b8d6-be9c8ef105b2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:36.598Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2b3ec8b6-519a-45f2-b8d6-be9c8ef105b2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"info","ts":"2026-09-18T10:08:36.598Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2b3ec8b6-519a-45f2-b8d6-be9c8ef105b2","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.598Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2b3ec8b6-519a-45f2-b8d6-be9c8ef105b2","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.598Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2b3ec8b6-519a-45f2-b8d6-be9c8ef105b2","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.598Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2b3ec8b6-519a-45f2-b8d6-be9c8ef105b2","name":"cluster-status-update-m6crj","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:08:36.599Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"56ae0fa9-9c99-4879-833e-8b93f02e980e","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.599Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"3497e81a-b4d1-4c65-9604-01ad559b3b79","namespacedName":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:36.599Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"3497e81a-b4d1-4c65-9604-01ad559b3b79","mode":"IntegratedOAuth","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.599Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"3497e81a-b4d1-4c65-9604-01ad559b3b79","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.600Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"e0b9d0ac-e644-4081-9503-2841e39f994d","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.600Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0d4bc07f-2c67-4b5f-b23c-3e6d945008ce","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:36.600Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0d4bc07f-2c67-4b5f-b23c-3e6d945008ce","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.600Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0d4bc07f-2c67-4b5f-b23c-3e6d945008ce","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:36.600Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0d4bc07f-2c67-4b5f-b23c-3e6d945008ce"}
{"level":"info","ts":"2026-09-18T10:08:36.600Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0d4bc07f-2c67-4b5f-b23c-3e6d945008ce","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:36.600Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0d4bc07f-2c67-4b5f-b23c-3e6d945008ce","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}}}
{"level":"info","ts":"2026-09-18T10:08:36.600Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0d4bc07f-2c67-4b5f-b23c-3e6d945008ce","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:08:36.605Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"84b75d51-0519-46f5-9a3a-6c865d1cfcec","rayCluster":"cluster-status-update-m6crj","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T10:08:36.605Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"84b75d51-0519-46f5-9a3a-6c865d1cfcec","rayCluster":"cluster-status-update-m6crj"}
{"level":"error","ts":"2026-09-18T10:08:36.607Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"3497e81a-b4d1-4c65-9604-01ad559b3b79","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"cluster-status-update-m6crj\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:36.607Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"3497e81a-b4d1-4c65-9604-01ad559b3b79"}
{"level":"error","ts":"2026-09-18T10:08:36.607Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"3497e81a-b4d1-4c65-9604-01ad559b3b79","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"cluster-status-update-m6crj\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:36.607Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2b3ec8b6-519a-45f2-b8d6-be9c8ef105b2","name":"cluster-status-update-m6crj","error":"Operation cannot be fulfilled on rayclusters.ray.io \"cluster-status-update-m6crj\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj"}}
{"level":"info","ts":"2026-09-18T10:08:36.607Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2b3ec8b6-519a-45f2-b8d6-be9c8ef105b2"}
{"level":"error","ts":"2026-09-18T10:08:36.607Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2b3ec8b6-519a-45f2-b8d6-be9c8ef105b2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:36.607Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"ba68020b-cf8c-4ea7-801e-82a35c5e207c"}
{"level":"info","ts":"2026-09-18T10:08:36.607Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"ba68020b-cf8c-4ea7-801e-82a35c5e207c"}
{"level":"info","ts":"2026-09-18T10:08:36.607Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"ba68020b-cf8c-4ea7-801e-82a35c5e207c"}
{"level":"info","ts":"2026-09-18T10:08:36.607Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"ba68020b-cf8c-4ea7-801e-82a35c5e207c","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:36.607Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"ba68020b-cf8c-4ea7-801e-82a35c5e207c","error":"head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"error","ts":"2026-09-18T10:08:36.607Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"ba68020b-cf8c-4ea7-801e-82a35c5e207c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:36.607Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"ba68020b-cf8c-4ea7-801e-82a35c5e207c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"info","ts":"2026-09-18T10:08:36.607Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"ba68020b-cf8c-4ea7-801e-82a35c5e207c","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.607Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"ba68020b-cf8c-4ea7-801e-82a35c5e207c","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.607Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"ba68020b-cf8c-4ea7-801e-82a35c5e207c","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.607Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"ba68020b-cf8c-4ea7-801e-82a35c5e207c","name":"cluster-status-update-m6crj","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:08:36.666Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"c49a52c8-3e00-456c-8dea-f3e79922e26d","namespacedName":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:36.666Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"c49a52c8-3e00-456c-8dea-f3e79922e26d","mode":"IntegratedOAuth","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.666Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"c49a52c8-3e00-456c-8dea-f3e79922e26d","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.667Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"84b75d51-0519-46f5-9a3a-6c865d1cfcec","rayCluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.668Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"aac50564-7778-4866-b94c-e46bedae8598","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:36.668Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"aac50564-7778-4866-b94c-e46bedae8598","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.668Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"aac50564-7778-4866-b94c-e46bedae8598","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:36.669Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"aac50564-7778-4866-b94c-e46bedae8598"}
{"level":"info","ts":"2026-09-18T10:08:36.669Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"aac50564-7778-4866-b94c-e46bedae8598","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:36.669Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"aac50564-7778-4866-b94c-e46bedae8598","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}}}
{"level":"info","ts":"2026-09-18T10:08:36.670Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"ba68020b-cf8c-4ea7-801e-82a35c5e207c"}
{"level":"error","ts":"2026-09-18T10:08:36.670Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"ba68020b-cf8c-4ea7-801e-82a35c5e207c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:36.670Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"b139f6df-582d-4fbc-ad9a-a10b84b2c43d"}
{"level":"info","ts":"2026-09-18T10:08:36.670Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"b139f6df-582d-4fbc-ad9a-a10b84b2c43d"}
{"level":"info","ts":"2026-09-18T10:08:36.670Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"b139f6df-582d-4fbc-ad9a-a10b84b2c43d"}
{"level":"info","ts":"2026-09-18T10:08:36.671Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"b139f6df-582d-4fbc-ad9a-a10b84b2c43d","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:36.671Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"b139f6df-582d-4fbc-ad9a-a10b84b2c43d","error":"head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"error","ts":"2026-09-18T10:08:36.671Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"b139f6df-582d-4fbc-ad9a-a10b84b2c43d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:36.673Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"b139f6df-582d-4fbc-ad9a-a10b84b2c43d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"info","ts":"2026-09-18T10:08:36.673Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"b139f6df-582d-4fbc-ad9a-a10b84b2c43d","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.673Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"b139f6df-582d-4fbc-ad9a-a10b84b2c43d","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.673Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"b139f6df-582d-4fbc-ad9a-a10b84b2c43d","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.673Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"b139f6df-582d-4fbc-ad9a-a10b84b2c43d"}
{"level":"error","ts":"2026-09-18T10:08:36.673Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"b139f6df-582d-4fbc-ad9a-a10b84b2c43d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:36.673Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"bc5f0890-7e2d-4371-9942-0ba62e7ae75c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:36.674Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"bc5f0890-7e2d-4371-9942-0ba62e7ae75c","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.674Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"bc5f0890-7e2d-4371-9942-0ba62e7ae75c","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:36.674Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"bc5f0890-7e2d-4371-9942-0ba62e7ae75c"}
{"level":"info","ts":"2026-09-18T10:08:36.674Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"bc5f0890-7e2d-4371-9942-0ba62e7ae75c","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:36.674Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"bc5f0890-7e2d-4371-9942-0ba62e7ae75c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:36.674Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"bc5f0890-7e2d-4371-9942-0ba62e7ae75c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:08:36.675Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"e0b9d0ac-e644-4081-9503-2841e39f994d","name":"cluster-status-update-m6crj-head"}
{"level":"error","ts":"2026-09-18T10:08:36.685Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"c49a52c8-3e00-456c-8dea-f3e79922e26d","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"cluster-status-update-m6crj\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:36.685Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"c49a52c8-3e00-456c-8dea-f3e79922e26d"}
{"level":"error","ts":"2026-09-18T10:08:36.765Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"c49a52c8-3e00-456c-8dea-f3e79922e26d","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"cluster-status-update-m6crj\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:36.766Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"31a43bb5-469d-4916-8885-d47da2155a36","namespacedName":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:36.766Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"31a43bb5-469d-4916-8885-d47da2155a36","mode":"IntegratedOAuth","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.766Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"31a43bb5-469d-4916-8885-d47da2155a36","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.691Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"35a1f68f-a71e-46e9-9b8f-947047e03d59"}
{"level":"info","ts":"2026-09-18T10:08:36.767Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"35a1f68f-a71e-46e9-9b8f-947047e03d59"}
{"level":"info","ts":"2026-09-18T10:08:36.767Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"35a1f68f-a71e-46e9-9b8f-947047e03d59"}
{"level":"info","ts":"2026-09-18T10:08:36.767Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"35a1f68f-a71e-46e9-9b8f-947047e03d59","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:36.767Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"35a1f68f-a71e-46e9-9b8f-947047e03d59","error":"head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"error","ts":"2026-09-18T10:08:36.767Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"35a1f68f-a71e-46e9-9b8f-947047e03d59","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:36.767Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"35a1f68f-a71e-46e9-9b8f-947047e03d59","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"info","ts":"2026-09-18T10:08:36.767Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"35a1f68f-a71e-46e9-9b8f-947047e03d59","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.767Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"35a1f68f-a71e-46e9-9b8f-947047e03d59","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.767Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"35a1f68f-a71e-46e9-9b8f-947047e03d59","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.767Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"35a1f68f-a71e-46e9-9b8f-947047e03d59"}
{"level":"error","ts":"2026-09-18T10:08:36.768Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"35a1f68f-a71e-46e9-9b8f-947047e03d59","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:36.769Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"e0b9d0ac-e644-4081-9503-2841e39f994d","name":"cluster-status-update-m6crj-workers"}
{"level":"info","ts":"2026-09-18T10:08:36.769Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"e0b9d0ac-e644-4081-9503-2841e39f994d","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.769Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"257603ef-5894-4042-898a-93eb65b3f174","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.771Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"94495e69-0788-4676-b2f7-84cc3ee49376","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:36.771Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"94495e69-0788-4676-b2f7-84cc3ee49376","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.771Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"94495e69-0788-4676-b2f7-84cc3ee49376","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:36.771Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"94495e69-0788-4676-b2f7-84cc3ee49376"}
{"level":"info","ts":"2026-09-18T10:08:36.771Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"94495e69-0788-4676-b2f7-84cc3ee49376","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:36.771Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"94495e69-0788-4676-b2f7-84cc3ee49376","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:36.771Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"84b75d51-0519-46f5-9a3a-6c865d1cfcec","rayCluster":"cluster-status-update-m6crj","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:08:36.780Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"31a43bb5-469d-4916-8885-d47da2155a36","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.792Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"84b75d51-0519-46f5-9a3a-6c865d1cfcec","rayCluster":"cluster-status-update-m6crj","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:08:36.792Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"84b75d51-0519-46f5-9a3a-6c865d1cfcec"}
{"level":"info","ts":"2026-09-18T10:08:36.792Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"4fe39417-632a-463e-9167-c3fe38e9c2da","rayCluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:36.792Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"4fe39417-632a-463e-9167-c3fe38e9c2da"}
{"level":"info","ts":"2026-09-18T10:08:36.866Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"257603ef-5894-4042-898a-93eb65b3f174","name":"cluster-status-update-m6crj-head"}
{"level":"info","ts":"2026-09-18T10:08:36.868Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"78136983-736f-40a5-9b4f-e2d23460a8ce"}
{"level":"info","ts":"2026-09-18T10:08:36.869Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"78136983-736f-40a5-9b4f-e2d23460a8ce"}
{"level":"info","ts":"2026-09-18T10:08:36.869Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"78136983-736f-40a5-9b4f-e2d23460a8ce"}
{"level":"info","ts":"2026-09-18T10:08:36.869Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"78136983-736f-40a5-9b4f-e2d23460a8ce","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:36.869Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"78136983-736f-40a5-9b4f-e2d23460a8ce","error":"head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"error","ts":"2026-09-18T10:08:36.869Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"78136983-736f-40a5-9b4f-e2d23460a8ce","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:36.869Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"78136983-736f-40a5-9b4f-e2d23460a8ce","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"info","ts":"2026-09-18T10:08:36.869Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"78136983-736f-40a5-9b4f-e2d23460a8ce","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.869Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"78136983-736f-40a5-9b4f-e2d23460a8ce","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.869Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"78136983-736f-40a5-9b4f-e2d23460a8ce","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:36.869Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"78136983-736f-40a5-9b4f-e2d23460a8ce"}
{"level":"error","ts":"2026-09-18T10:08:36.869Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"78136983-736f-40a5-9b4f-e2d23460a8ce","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:36.893Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"257603ef-5894-4042-898a-93eb65b3f174","name":"cluster-status-update-m6crj-workers"}
{"level":"info","ts":"2026-09-18T10:08:36.893Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"257603ef-5894-4042-898a-93eb65b3f174","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:37.030Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"12d9cdca-e95d-4203-a93c-e299c90f3e57"}
{"level":"info","ts":"2026-09-18T10:08:37.030Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"12d9cdca-e95d-4203-a93c-e299c90f3e57"}
{"level":"info","ts":"2026-09-18T10:08:37.030Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"12d9cdca-e95d-4203-a93c-e299c90f3e57"}
{"level":"info","ts":"2026-09-18T10:08:37.030Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"12d9cdca-e95d-4203-a93c-e299c90f3e57","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:37.030Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"12d9cdca-e95d-4203-a93c-e299c90f3e57","error":"head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"error","ts":"2026-09-18T10:08:37.030Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"12d9cdca-e95d-4203-a93c-e299c90f3e57","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:37.030Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"12d9cdca-e95d-4203-a93c-e299c90f3e57","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"info","ts":"2026-09-18T10:08:37.031Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"12d9cdca-e95d-4203-a93c-e299c90f3e57","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:37.031Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"12d9cdca-e95d-4203-a93c-e299c90f3e57","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:37.031Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"12d9cdca-e95d-4203-a93c-e299c90f3e57","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:37.031Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"12d9cdca-e95d-4203-a93c-e299c90f3e57"}
{"level":"error","ts":"2026-09-18T10:08:37.031Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"12d9cdca-e95d-4203-a93c-e299c90f3e57","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:37.351Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7621faaa-628a-4377-9cd5-e4b5646b15f5"}
{"level":"info","ts":"2026-09-18T10:08:37.351Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7621faaa-628a-4377-9cd5-e4b5646b15f5"}
{"level":"info","ts":"2026-09-18T10:08:37.351Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7621faaa-628a-4377-9cd5-e4b5646b15f5"}
{"level":"info","ts":"2026-09-18T10:08:37.351Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7621faaa-628a-4377-9cd5-e4b5646b15f5","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:37.351Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7621faaa-628a-4377-9cd5-e4b5646b15f5","error":"head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"error","ts":"2026-09-18T10:08:37.351Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7621faaa-628a-4377-9cd5-e4b5646b15f5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:37.351Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7621faaa-628a-4377-9cd5-e4b5646b15f5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"info","ts":"2026-09-18T10:08:37.351Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7621faaa-628a-4377-9cd5-e4b5646b15f5","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:37.351Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7621faaa-628a-4377-9cd5-e4b5646b15f5","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:37.351Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7621faaa-628a-4377-9cd5-e4b5646b15f5","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:37.351Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7621faaa-628a-4377-9cd5-e4b5646b15f5"}
{"level":"error","ts":"2026-09-18T10:08:37.351Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7621faaa-628a-4377-9cd5-e4b5646b15f5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:37.992Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"0a87b1ff-c221-4bb3-849d-f01c8fc6c047"}
{"level":"info","ts":"2026-09-18T10:08:37.992Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"0a87b1ff-c221-4bb3-849d-f01c8fc6c047"}
{"level":"info","ts":"2026-09-18T10:08:37.992Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"0a87b1ff-c221-4bb3-849d-f01c8fc6c047"}
{"level":"info","ts":"2026-09-18T10:08:37.992Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"0a87b1ff-c221-4bb3-849d-f01c8fc6c047","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:37.992Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"0a87b1ff-c221-4bb3-849d-f01c8fc6c047","error":"head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"error","ts":"2026-09-18T10:08:37.992Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"0a87b1ff-c221-4bb3-849d-f01c8fc6c047","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:37.992Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"0a87b1ff-c221-4bb3-849d-f01c8fc6c047","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found"}
{"level":"info","ts":"2026-09-18T10:08:37.992Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"0a87b1ff-c221-4bb3-849d-f01c8fc6c047","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:37.992Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"0a87b1ff-c221-4bb3-849d-f01c8fc6c047","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:37.993Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"0a87b1ff-c221-4bb3-849d-f01c8fc6c047","filter labels":["test-ns-ltx56",{"ray.io/cluster":"cluster-status-update-m6crj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:08:37.993Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"0a87b1ff-c221-4bb3-849d-f01c8fc6c047"}
{"level":"error","ts":"2026-09-18T10:08:37.993Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"0a87b1ff-c221-4bb3-849d-f01c8fc6c047","error":"mTLS secrets not ready: head certificate secret ray-head-secret-cluster-status-update-m6crj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:38.737Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"aada609c-2524-4f58-8ab3-295382d81fb7","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:38.737Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"aada609c-2524-4f58-8ab3-295382d81fb7","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:08:35 +0000 UTC","Now":"2026-09-18T10:08:38.737Z","ShutdownTime":"2026-09-18T10:08:35.000Z"}
{"level":"info","ts":"2026-09-18T10:08:38.737Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"aada609c-2524-4f58-8ab3-295382d81fb7","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:38.737Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"aada609c-2524-4f58-8ab3-295382d81fb7","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T10:08:38.737Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"aada609c-2524-4f58-8ab3-295382d81fb7","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:08:39.273Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2"}
{"level":"info","ts":"2026-09-18T10:08:39.273Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2"}
{"level":"info","ts":"2026-09-18T10:08:39.273Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2"}
{"level":"info","ts":"2026-09-18T10:08:39.274Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:39.274Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2"}
{"level":"info","ts":"2026-09-18T10:08:39.274Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:08:39.274Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2"}
{"level":"info","ts":"2026-09-18T10:08:39.274Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","mode":"IntegratedOAuth","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:39.274Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","cluster":"cluster-status-update-m6crj","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:08:39.274Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","cluster":"cluster-status-update-m6crj","authType":"IntegratedOAuth","serviceAccount":"cluster-status-update-m6crj-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T10:08:39.274Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"cluster-status-update-m6crj","ray.io/group":"headgroup","ray.io/identifier":"cluster-status-update-m6crj-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:08:39.274Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T10:08:39.274Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:08:39.274Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:08:39.274Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","enabled":true}
{"level":"info","ts":"2026-09-18T10:08:39.301Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","name":"cluster-status-update-m6crj-head-9fc7t"}
{"level":"info","ts":"2026-09-18T10:08:39.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:39.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:39.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:08:39.301Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:08:39.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:39.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:08:39.301Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2"}
{"level":"info","ts":"2026-09-18T10:08:39.301Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","nodeType":"worker","rayStartParams":{"address":"cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:08:39.301Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","rayStartCmd":"ray start  --address=cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:08:39.301Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:08:39.301Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","enabled":true}
{"level":"info","ts":"2026-09-18T10:08:39.324Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","name":"cluster-status-update-m6crj-small-group-worker-5l8bd"}
{"level":"info","ts":"2026-09-18T10:08:39.324Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:39.324Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:39.325Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"80538d1d-8a07-4f1f-bf67-2c8d2e8a4bf2","name":"cluster-status-update-m6crj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:08:39.343Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b"}
{"level":"info","ts":"2026-09-18T10:08:39.343Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b"}
{"level":"info","ts":"2026-09-18T10:08:39.343Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b"}
{"level":"info","ts":"2026-09-18T10:08:39.343Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:39.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:39.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:39.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:39.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:39.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:39.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:39.343Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:39.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:39.343Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:39.343Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:39.343Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b","name":"cluster-status-update-m6crj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:08:39.344Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"40af43f0-30d5-408d-84b8-969a2047f7ef","rayCluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:39.344Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"7bcaeae3-f2d2-4dfb-b703-41d2bfa344c1","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:39.344Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"40af43f0-30d5-408d-84b8-969a2047f7ef","rayCluster":"cluster-status-update-m6crj","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:08:39.345Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"c19ab8e1-a011-4879-8d25-d7e0932a9153","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:39.345Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"c19ab8e1-a011-4879-8d25-d7e0932a9153","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:39.345Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"c19ab8e1-a011-4879-8d25-d7e0932a9153","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:39.345Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"c19ab8e1-a011-4879-8d25-d7e0932a9153"}
{"level":"info","ts":"2026-09-18T10:08:39.345Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"c19ab8e1-a011-4879-8d25-d7e0932a9153","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:39.345Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"c19ab8e1-a011-4879-8d25-d7e0932a9153","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:39.345Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"c19ab8e1-a011-4879-8d25-d7e0932a9153","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:08:39.368Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"7bcaeae3-f2d2-4dfb-b703-41d2bfa344c1","name":"cluster-status-update-m6crj-head"}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b","name":"cluster-status-update-m6crj","error":"Operation cannot be fulfilled on rayclusters.ray.io \"cluster-status-update-m6crj\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj"}}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b"}
{"level":"error","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5915e667-651d-429d-970c-3a1bf1bac32b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"cluster-status-update-m6crj\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6df9c08d-2fcf-4f4b-87e4-efe2b1d874f9"}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6df9c08d-2fcf-4f4b-87e4-efe2b1d874f9"}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6df9c08d-2fcf-4f4b-87e4-efe2b1d874f9"}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6df9c08d-2fcf-4f4b-87e4-efe2b1d874f9","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6df9c08d-2fcf-4f4b-87e4-efe2b1d874f9","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6df9c08d-2fcf-4f4b-87e4-efe2b1d874f9","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6df9c08d-2fcf-4f4b-87e4-efe2b1d874f9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6df9c08d-2fcf-4f4b-87e4-efe2b1d874f9","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6df9c08d-2fcf-4f4b-87e4-efe2b1d874f9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6df9c08d-2fcf-4f4b-87e4-efe2b1d874f9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6df9c08d-2fcf-4f4b-87e4-efe2b1d874f9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6df9c08d-2fcf-4f4b-87e4-efe2b1d874f9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6df9c08d-2fcf-4f4b-87e4-efe2b1d874f9","podName":"cluster-status-update-m6crj-small-group-worker-5l8bd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6df9c08d-2fcf-4f4b-87e4-efe2b1d874f9","podName":"cluster-status-update-m6crj-small-group-worker-5l8bd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:39.373Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6df9c08d-2fcf-4f4b-87e4-efe2b1d874f9","name":"cluster-status-update-m6crj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:08:39.374Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"12d0a2f2-9e7f-48da-8125-2a5e72558021","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:39.374Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"12d0a2f2-9e7f-48da-8125-2a5e72558021","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:39.375Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"12d0a2f2-9e7f-48da-8125-2a5e72558021","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:39.375Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"12d0a2f2-9e7f-48da-8125-2a5e72558021"}
{"level":"info","ts":"2026-09-18T10:08:39.375Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"12d0a2f2-9e7f-48da-8125-2a5e72558021","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:39.375Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"12d0a2f2-9e7f-48da-8125-2a5e72558021","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:39.387Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"7bcaeae3-f2d2-4dfb-b703-41d2bfa344c1","name":"cluster-status-update-m6crj-workers"}
{"level":"info","ts":"2026-09-18T10:08:39.387Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"7bcaeae3-f2d2-4dfb-b703-41d2bfa344c1","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"711859d8-4d61-411e-b57d-a804e4b8a909","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"711859d8-4d61-411e-b57d-a804e4b8a909","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"711859d8-4d61-411e-b57d-a804e4b8a909","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"4ace30c3-195f-4cef-b351-af7e31a75ab1"}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"4ace30c3-195f-4cef-b351-af7e31a75ab1"}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"4ace30c3-195f-4cef-b351-af7e31a75ab1"}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"4ace30c3-195f-4cef-b351-af7e31a75ab1","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"22f79798-4f42-401b-b25e-9286120817f6","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"4ace30c3-195f-4cef-b351-af7e31a75ab1","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"4ace30c3-195f-4cef-b351-af7e31a75ab1","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"4ace30c3-195f-4cef-b351-af7e31a75ab1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"4ace30c3-195f-4cef-b351-af7e31a75ab1","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"4ace30c3-195f-4cef-b351-af7e31a75ab1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"4ace30c3-195f-4cef-b351-af7e31a75ab1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"4ace30c3-195f-4cef-b351-af7e31a75ab1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"4ace30c3-195f-4cef-b351-af7e31a75ab1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"711859d8-4d61-411e-b57d-a804e4b8a909"}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"711859d8-4d61-411e-b57d-a804e4b8a909","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"4ace30c3-195f-4cef-b351-af7e31a75ab1","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"711859d8-4d61-411e-b57d-a804e4b8a909","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"4ace30c3-195f-4cef-b351-af7e31a75ab1","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"711859d8-4d61-411e-b57d-a804e4b8a909","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"4ace30c3-195f-4cef-b351-af7e31a75ab1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"4ace30c3-195f-4cef-b351-af7e31a75ab1","seconds":300}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"e4c1ba68-2e8c-482d-9eb8-0127d589fa15","rayCluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:39.389Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"e4c1ba68-2e8c-482d-9eb8-0127d589fa15","rayCluster":"cluster-status-update-m6crj","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:08:39.401Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"22f79798-4f42-401b-b25e-9286120817f6","name":"cluster-status-update-m6crj-head"}
{"level":"info","ts":"2026-09-18T10:08:39.466Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"22f79798-4f42-401b-b25e-9286120817f6","name":"cluster-status-update-m6crj-workers"}
{"level":"info","ts":"2026-09-18T10:08:39.466Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"22f79798-4f42-401b-b25e-9286120817f6","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:39.466Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"63e718f9-41ab-4d6d-9d64-0f329ac18374","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:39.466Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"63e718f9-41ab-4d6d-9d64-0f329ac18374","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:39.466Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"63e718f9-41ab-4d6d-9d64-0f329ac18374","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:39.466Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"63e718f9-41ab-4d6d-9d64-0f329ac18374"}
{"level":"info","ts":"2026-09-18T10:08:39.466Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"63e718f9-41ab-4d6d-9d64-0f329ac18374","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:39.466Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"63e718f9-41ab-4d6d-9d64-0f329ac18374","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:39.504Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fa0b8040-2da6-4282-a100-0ff2305c5497","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:39.504Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fa0b8040-2da6-4282-a100-0ff2305c5497","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:39.504Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fa0b8040-2da6-4282-a100-0ff2305c5497","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:39.504Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fa0b8040-2da6-4282-a100-0ff2305c5497"}
{"level":"info","ts":"2026-09-18T10:08:39.504Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fa0b8040-2da6-4282-a100-0ff2305c5497","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:39.504Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fa0b8040-2da6-4282-a100-0ff2305c5497","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:39.905Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cad90b01-3006-44aa-a540-c559595a2646"}
{"level":"info","ts":"2026-09-18T10:08:39.905Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cad90b01-3006-44aa-a540-c559595a2646"}
{"level":"info","ts":"2026-09-18T10:08:39.905Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cad90b01-3006-44aa-a540-c559595a2646"}
{"level":"info","ts":"2026-09-18T10:08:39.905Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cad90b01-3006-44aa-a540-c559595a2646","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:39.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cad90b01-3006-44aa-a540-c559595a2646","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:39.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cad90b01-3006-44aa-a540-c559595a2646","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:39.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cad90b01-3006-44aa-a540-c559595a2646","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:39.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cad90b01-3006-44aa-a540-c559595a2646","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:39.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cad90b01-3006-44aa-a540-c559595a2646","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:39.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cad90b01-3006-44aa-a540-c559595a2646","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:39.906Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cad90b01-3006-44aa-a540-c559595a2646","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:39.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cad90b01-3006-44aa-a540-c559595a2646","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:39.906Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cad90b01-3006-44aa-a540-c559595a2646","podName":"cluster-status-update-m6crj-small-group-worker-5l8bd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:39.906Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cad90b01-3006-44aa-a540-c559595a2646","podName":"cluster-status-update-m6crj-small-group-worker-5l8bd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:39.906Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cad90b01-3006-44aa-a540-c559595a2646","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:08:39.906Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cad90b01-3006-44aa-a540-c559595a2646","seconds":300}
{"level":"info","ts":"2026-09-18T10:08:39.931Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"85448c38-1c94-4362-a2a9-f001940358b9"}
{"level":"info","ts":"2026-09-18T10:08:39.931Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"85448c38-1c94-4362-a2a9-f001940358b9"}
{"level":"info","ts":"2026-09-18T10:08:39.931Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"85448c38-1c94-4362-a2a9-f001940358b9"}
{"level":"info","ts":"2026-09-18T10:08:39.931Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"85448c38-1c94-4362-a2a9-f001940358b9","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:39.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"85448c38-1c94-4362-a2a9-f001940358b9","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:39.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"85448c38-1c94-4362-a2a9-f001940358b9","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:39.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"85448c38-1c94-4362-a2a9-f001940358b9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:39.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"85448c38-1c94-4362-a2a9-f001940358b9","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:39.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"85448c38-1c94-4362-a2a9-f001940358b9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:39.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"85448c38-1c94-4362-a2a9-f001940358b9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:39.932Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"85448c38-1c94-4362-a2a9-f001940358b9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:39.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"85448c38-1c94-4362-a2a9-f001940358b9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:39.932Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"85448c38-1c94-4362-a2a9-f001940358b9","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:39.932Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"85448c38-1c94-4362-a2a9-f001940358b9","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:39.932Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"85448c38-1c94-4362-a2a9-f001940358b9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:08:39.932Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"85448c38-1c94-4362-a2a9-f001940358b9","seconds":300}
{"level":"info","ts":"2026-09-18T10:08:40.033Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"993d98d2-8dd8-4ee1-bcd3-7bd24cfc46f6"}
{"level":"info","ts":"2026-09-18T10:08:40.033Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"993d98d2-8dd8-4ee1-bcd3-7bd24cfc46f6"}
{"level":"info","ts":"2026-09-18T10:08:40.033Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"993d98d2-8dd8-4ee1-bcd3-7bd24cfc46f6"}
{"level":"info","ts":"2026-09-18T10:08:40.033Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"993d98d2-8dd8-4ee1-bcd3-7bd24cfc46f6","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:40.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"993d98d2-8dd8-4ee1-bcd3-7bd24cfc46f6","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:40.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"993d98d2-8dd8-4ee1-bcd3-7bd24cfc46f6","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:40.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"993d98d2-8dd8-4ee1-bcd3-7bd24cfc46f6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:40.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"993d98d2-8dd8-4ee1-bcd3-7bd24cfc46f6","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:40.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"993d98d2-8dd8-4ee1-bcd3-7bd24cfc46f6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:40.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"993d98d2-8dd8-4ee1-bcd3-7bd24cfc46f6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:40.033Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"993d98d2-8dd8-4ee1-bcd3-7bd24cfc46f6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:40.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"993d98d2-8dd8-4ee1-bcd3-7bd24cfc46f6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:40.033Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"993d98d2-8dd8-4ee1-bcd3-7bd24cfc46f6","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:40.033Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"993d98d2-8dd8-4ee1-bcd3-7bd24cfc46f6","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:40.033Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"993d98d2-8dd8-4ee1-bcd3-7bd24cfc46f6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:08:40.033Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"993d98d2-8dd8-4ee1-bcd3-7bd24cfc46f6","seconds":300}
{"level":"info","ts":"2026-09-18T10:08:40.815Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5e19625a-b7a5-4c64-bc56-6191767e9663"}
{"level":"info","ts":"2026-09-18T10:08:40.815Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5e19625a-b7a5-4c64-bc56-6191767e9663"}
{"level":"info","ts":"2026-09-18T10:08:40.815Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5e19625a-b7a5-4c64-bc56-6191767e9663"}
{"level":"info","ts":"2026-09-18T10:08:40.815Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5e19625a-b7a5-4c64-bc56-6191767e9663","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:40.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5e19625a-b7a5-4c64-bc56-6191767e9663","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:40.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5e19625a-b7a5-4c64-bc56-6191767e9663","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:40.816Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5e19625a-b7a5-4c64-bc56-6191767e9663","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:40.816Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5e19625a-b7a5-4c64-bc56-6191767e9663","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:40.816Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5e19625a-b7a5-4c64-bc56-6191767e9663","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:40.816Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5e19625a-b7a5-4c64-bc56-6191767e9663","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:40.816Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5e19625a-b7a5-4c64-bc56-6191767e9663","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:40.816Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5e19625a-b7a5-4c64-bc56-6191767e9663","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:40.816Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5e19625a-b7a5-4c64-bc56-6191767e9663","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:40.816Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5e19625a-b7a5-4c64-bc56-6191767e9663","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:40.816Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5e19625a-b7a5-4c64-bc56-6191767e9663","name":"cluster-status-update-m6crj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:08:40.833Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"7e1afee0-2fc8-4fac-b7f1-5448752374ac","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:40.834Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"37a2155c-f376-453f-92be-dcf1d95a7a74","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:40.834Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"37a2155c-f376-453f-92be-dcf1d95a7a74","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:40.834Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"37a2155c-f376-453f-92be-dcf1d95a7a74","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:40.834Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"bb1b92cb-5b26-4c79-9c7e-df117c21ddb9","rayCluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:40.834Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"37a2155c-f376-453f-92be-dcf1d95a7a74"}
{"level":"info","ts":"2026-09-18T10:08:40.834Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"37a2155c-f376-453f-92be-dcf1d95a7a74","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:40.834Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"bb1b92cb-5b26-4c79-9c7e-df117c21ddb9","pod":"cluster-status-update-m6crj-head-9fc7t","ip":"10.132.0.54"}
{"level":"info","ts":"2026-09-18T10:08:40.834Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"bb1b92cb-5b26-4c79-9c7e-df117c21ddb9","pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","ip":"10.133.0.46"}
{"level":"info","ts":"2026-09-18T10:08:40.834Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"37a2155c-f376-453f-92be-dcf1d95a7a74","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:40.834Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"37a2155c-f376-453f-92be-dcf1d95a7a74","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:08:40.847Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"bb1b92cb-5b26-4c79-9c7e-df117c21ddb9","certificate":"ray-head-cert-cluster-status-update-m6crj","dnsNames":["cluster-status-update-m6crj-head-svc","localhost","cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc","cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.54","10.133.0.46","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:08:40.848Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"7e1afee0-2fc8-4fac-b7f1-5448752374ac","name":"cluster-status-update-m6crj-head"}
{"level":"info","ts":"2026-09-18T10:08:40.850Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"2cde49e2-92e4-4abb-90ae-6936918d5558","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:40.850Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"2cde49e2-92e4-4abb-90ae-6936918d5558","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:40.850Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"2cde49e2-92e4-4abb-90ae-6936918d5558","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:40.850Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"2cde49e2-92e4-4abb-90ae-6936918d5558"}
{"level":"info","ts":"2026-09-18T10:08:40.850Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"2cde49e2-92e4-4abb-90ae-6936918d5558","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:40.850Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"2cde49e2-92e4-4abb-90ae-6936918d5558","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:40.861Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"bb1b92cb-5b26-4c79-9c7e-df117c21ddb9","certificate":"ray-worker-cert-cluster-status-update-m6crj","dnsNames":["cluster-status-update-m6crj-worker-svc","localhost","cluster-status-update-m6crj-small-group","cluster-status-update-m6crj-worker-svc.test-ns-ltx56.svc","cluster-status-update-m6crj-worker-svc.test-ns-ltx56.svc.cluster.local","*.cluster-status-update-m6crj-worker-svc.test-ns-ltx56.svc","*.cluster-status-update-m6crj-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.54","10.133.0.46","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:08:40.861Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"bb1b92cb-5b26-4c79-9c7e-df117c21ddb9","rayCluster":"cluster-status-update-m6crj","podIPs":["10.132.0.54","10.133.0.46"]}
{"level":"info","ts":"2026-09-18T10:08:40.863Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"7e1afee0-2fc8-4fac-b7f1-5448752374ac","name":"cluster-status-update-m6crj-workers"}
{"level":"info","ts":"2026-09-18T10:08:40.863Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"7e1afee0-2fc8-4fac-b7f1-5448752374ac","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:41.389Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"9e58ac0e-73d4-480f-8614-c6ca204a5b50"}
{"level":"info","ts":"2026-09-18T10:08:41.389Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"9e58ac0e-73d4-480f-8614-c6ca204a5b50"}
{"level":"info","ts":"2026-09-18T10:08:41.390Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"9e58ac0e-73d4-480f-8614-c6ca204a5b50"}
{"level":"info","ts":"2026-09-18T10:08:41.390Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"9e58ac0e-73d4-480f-8614-c6ca204a5b50","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:41.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"9e58ac0e-73d4-480f-8614-c6ca204a5b50","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:41.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"9e58ac0e-73d4-480f-8614-c6ca204a5b50","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:41.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"9e58ac0e-73d4-480f-8614-c6ca204a5b50","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:41.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"9e58ac0e-73d4-480f-8614-c6ca204a5b50","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:41.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"9e58ac0e-73d4-480f-8614-c6ca204a5b50","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:41.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"9e58ac0e-73d4-480f-8614-c6ca204a5b50","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:41.390Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"9e58ac0e-73d4-480f-8614-c6ca204a5b50","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:41.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"9e58ac0e-73d4-480f-8614-c6ca204a5b50","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:41.390Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"9e58ac0e-73d4-480f-8614-c6ca204a5b50","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:41.390Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"9e58ac0e-73d4-480f-8614-c6ca204a5b50","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:41.390Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"9e58ac0e-73d4-480f-8614-c6ca204a5b50","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:08:41.390Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"9e58ac0e-73d4-480f-8614-c6ca204a5b50","seconds":300}
{"level":"info","ts":"2026-09-18T10:08:42.505Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"438559ef-717f-4285-a9c4-876c2d4b51f1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:42.505Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"438559ef-717f-4285-a9c4-876c2d4b51f1","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:42.505Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"438559ef-717f-4285-a9c4-876c2d4b51f1","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:42.505Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"438559ef-717f-4285-a9c4-876c2d4b51f1"}
{"level":"info","ts":"2026-09-18T10:08:42.505Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"438559ef-717f-4285-a9c4-876c2d4b51f1","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:42.505Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"438559ef-717f-4285-a9c4-876c2d4b51f1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:45.506Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"b69facf9-ee28-4499-a464-cb1c3af39968","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:45.506Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"b69facf9-ee28-4499-a464-cb1c3af39968","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:45.506Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"b69facf9-ee28-4499-a464-cb1c3af39968","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:45.506Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"b69facf9-ee28-4499-a464-cb1c3af39968"}
{"level":"info","ts":"2026-09-18T10:08:45.506Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"b69facf9-ee28-4499-a464-cb1c3af39968","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:45.506Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"b69facf9-ee28-4499-a464-cb1c3af39968","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:45.582Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"fc241f93-5f91-496e-b93c-ff9f8b63c305","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:08:45.582Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"fc241f93-5f91-496e-b93c-ff9f8b63c305","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:08:45.582Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"fc241f93-5f91-496e-b93c-ff9f8b63c305","pod":"long-running-5qfl8-small-group-worker-tbqtj","ip":"10.132.0.47"}
{"level":"info","ts":"2026-09-18T10:08:45.582Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"fc241f93-5f91-496e-b93c-ff9f8b63c305","rayCluster":"long-running-5qfl8","podIPs":["10.133.0.42","10.132.0.47"]}
{"level":"info","ts":"2026-09-18T10:08:45.852Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"659e4408-da20-462c-b810-cb208b3a8c00"}
{"level":"info","ts":"2026-09-18T10:08:45.852Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"659e4408-da20-462c-b810-cb208b3a8c00"}
{"level":"info","ts":"2026-09-18T10:08:45.852Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"659e4408-da20-462c-b810-cb208b3a8c00"}
{"level":"info","ts":"2026-09-18T10:08:45.852Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"659e4408-da20-462c-b810-cb208b3a8c00","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:45.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"659e4408-da20-462c-b810-cb208b3a8c00","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:45.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"659e4408-da20-462c-b810-cb208b3a8c00","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:45.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"659e4408-da20-462c-b810-cb208b3a8c00","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:45.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"659e4408-da20-462c-b810-cb208b3a8c00","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:45.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"659e4408-da20-462c-b810-cb208b3a8c00","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:45.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"659e4408-da20-462c-b810-cb208b3a8c00","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:45.853Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"659e4408-da20-462c-b810-cb208b3a8c00","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:45.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"659e4408-da20-462c-b810-cb208b3a8c00","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:45.853Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"659e4408-da20-462c-b810-cb208b3a8c00","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:45.853Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"659e4408-da20-462c-b810-cb208b3a8c00","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:45.853Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"659e4408-da20-462c-b810-cb208b3a8c00","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:08:45.853Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"659e4408-da20-462c-b810-cb208b3a8c00","seconds":300}
{"level":"info","ts":"2026-09-18T10:08:46.063Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"883422cb-43f0-43e8-848b-a4ffc38f5020"}
{"level":"info","ts":"2026-09-18T10:08:46.063Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"883422cb-43f0-43e8-848b-a4ffc38f5020"}
{"level":"info","ts":"2026-09-18T10:08:46.063Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"883422cb-43f0-43e8-848b-a4ffc38f5020"}
{"level":"info","ts":"2026-09-18T10:08:46.063Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"883422cb-43f0-43e8-848b-a4ffc38f5020","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:46.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"883422cb-43f0-43e8-848b-a4ffc38f5020","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:46.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"883422cb-43f0-43e8-848b-a4ffc38f5020","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:46.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"883422cb-43f0-43e8-848b-a4ffc38f5020","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:46.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"883422cb-43f0-43e8-848b-a4ffc38f5020","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:46.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"883422cb-43f0-43e8-848b-a4ffc38f5020","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:46.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"883422cb-43f0-43e8-848b-a4ffc38f5020","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:46.063Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"883422cb-43f0-43e8-848b-a4ffc38f5020","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:46.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"883422cb-43f0-43e8-848b-a4ffc38f5020","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:46.063Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"883422cb-43f0-43e8-848b-a4ffc38f5020","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:46.063Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"883422cb-43f0-43e8-848b-a4ffc38f5020","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:46.063Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"883422cb-43f0-43e8-848b-a4ffc38f5020","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:08:46.063Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"883422cb-43f0-43e8-848b-a4ffc38f5020","seconds":300}
{"level":"info","ts":"2026-09-18T10:08:46.850Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5c6c3166-827e-40dc-8c84-395da73f9980"}
{"level":"info","ts":"2026-09-18T10:08:46.850Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5c6c3166-827e-40dc-8c84-395da73f9980"}
{"level":"info","ts":"2026-09-18T10:08:46.850Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5c6c3166-827e-40dc-8c84-395da73f9980"}
{"level":"info","ts":"2026-09-18T10:08:46.850Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5c6c3166-827e-40dc-8c84-395da73f9980","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:46.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5c6c3166-827e-40dc-8c84-395da73f9980","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:46.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5c6c3166-827e-40dc-8c84-395da73f9980","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:46.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5c6c3166-827e-40dc-8c84-395da73f9980","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:46.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5c6c3166-827e-40dc-8c84-395da73f9980","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:46.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5c6c3166-827e-40dc-8c84-395da73f9980","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:46.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5c6c3166-827e-40dc-8c84-395da73f9980","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:46.850Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5c6c3166-827e-40dc-8c84-395da73f9980","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:46.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5c6c3166-827e-40dc-8c84-395da73f9980","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:46.850Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5c6c3166-827e-40dc-8c84-395da73f9980","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:08:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:08:46.850Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5c6c3166-827e-40dc-8c84-395da73f9980","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:08:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:08:46.850Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"5c6c3166-827e-40dc-8c84-395da73f9980","name":"cluster-status-update-m6crj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:08:46.870Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"ab473fbe-ace4-47cb-aafd-d70481c60b36","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:46.870Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"37832ff1-0bf1-4ab0-b5ae-968f04c28b8b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:46.870Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"37832ff1-0bf1-4ab0-b5ae-968f04c28b8b","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:46.870Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"37832ff1-0bf1-4ab0-b5ae-968f04c28b8b","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:46.870Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"37832ff1-0bf1-4ab0-b5ae-968f04c28b8b"}
{"level":"info","ts":"2026-09-18T10:08:46.870Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"37832ff1-0bf1-4ab0-b5ae-968f04c28b8b","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:46.870Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"37832ff1-0bf1-4ab0-b5ae-968f04c28b8b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:46.870Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"37832ff1-0bf1-4ab0-b5ae-968f04c28b8b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:08:46.870Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"869b1f37-dc4e-47a2-a79c-5170c49bd213","rayCluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:46.871Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"869b1f37-dc4e-47a2-a79c-5170c49bd213","pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","ip":"10.133.0.46"}
{"level":"info","ts":"2026-09-18T10:08:46.871Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"869b1f37-dc4e-47a2-a79c-5170c49bd213","pod":"cluster-status-update-m6crj-head-9fc7t","ip":"10.132.0.54"}
{"level":"info","ts":"2026-09-18T10:08:46.871Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"869b1f37-dc4e-47a2-a79c-5170c49bd213","rayCluster":"cluster-status-update-m6crj","podIPs":["10.133.0.46","10.132.0.54"]}
{"level":"info","ts":"2026-09-18T10:08:46.884Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"ab473fbe-ace4-47cb-aafd-d70481c60b36","name":"cluster-status-update-m6crj-head"}
{"level":"info","ts":"2026-09-18T10:08:46.887Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"095b802e-0d57-4434-9003-b465812ae736","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:46.887Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"095b802e-0d57-4434-9003-b465812ae736","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:46.887Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"095b802e-0d57-4434-9003-b465812ae736","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:46.888Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"095b802e-0d57-4434-9003-b465812ae736"}
{"level":"info","ts":"2026-09-18T10:08:46.888Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"095b802e-0d57-4434-9003-b465812ae736","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:46.888Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"095b802e-0d57-4434-9003-b465812ae736","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:46.903Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"ab473fbe-ace4-47cb-aafd-d70481c60b36","name":"cluster-status-update-m6crj-workers"}
{"level":"info","ts":"2026-09-18T10:08:46.903Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"ab473fbe-ace4-47cb-aafd-d70481c60b36","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:47.077Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"1077e5b0-ab38-4a70-afcd-60753b301a01"}
{"level":"info","ts":"2026-09-18T10:08:47.077Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"1077e5b0-ab38-4a70-afcd-60753b301a01"}
{"level":"info","ts":"2026-09-18T10:08:47.077Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"1077e5b0-ab38-4a70-afcd-60753b301a01"}
{"level":"info","ts":"2026-09-18T10:08:47.078Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"1077e5b0-ab38-4a70-afcd-60753b301a01","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:47.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"1077e5b0-ab38-4a70-afcd-60753b301a01","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:47.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"1077e5b0-ab38-4a70-afcd-60753b301a01","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:47.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"1077e5b0-ab38-4a70-afcd-60753b301a01","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:47.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"1077e5b0-ab38-4a70-afcd-60753b301a01","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:47.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"1077e5b0-ab38-4a70-afcd-60753b301a01","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:47.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"1077e5b0-ab38-4a70-afcd-60753b301a01","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:47.078Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"1077e5b0-ab38-4a70-afcd-60753b301a01","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:47.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"1077e5b0-ab38-4a70-afcd-60753b301a01","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:47.078Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"1077e5b0-ab38-4a70-afcd-60753b301a01","podName":"cluster-status-update-m6crj-small-group-worker-5l8bd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:47.078Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"1077e5b0-ab38-4a70-afcd-60753b301a01","podName":"cluster-status-update-m6crj-small-group-worker-5l8bd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:47.078Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"1077e5b0-ab38-4a70-afcd-60753b301a01","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:08:47.078Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"1077e5b0-ab38-4a70-afcd-60753b301a01","seconds":300}
{"level":"info","ts":"2026-09-18T10:08:48.507Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"bfd59af8-d6f0-46fe-90c2-9777b084f09b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:48.507Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"bfd59af8-d6f0-46fe-90c2-9777b084f09b","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:48.507Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"bfd59af8-d6f0-46fe-90c2-9777b084f09b","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:48.507Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"bfd59af8-d6f0-46fe-90c2-9777b084f09b"}
{"level":"info","ts":"2026-09-18T10:08:48.507Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"bfd59af8-d6f0-46fe-90c2-9777b084f09b","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:48.507Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"bfd59af8-d6f0-46fe-90c2-9777b084f09b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:48.870Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7047a819-e8ba-4e63-b0aa-67f9966fae62"}
{"level":"info","ts":"2026-09-18T10:08:48.870Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7047a819-e8ba-4e63-b0aa-67f9966fae62"}
{"level":"info","ts":"2026-09-18T10:08:48.870Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7047a819-e8ba-4e63-b0aa-67f9966fae62"}
{"level":"info","ts":"2026-09-18T10:08:48.870Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7047a819-e8ba-4e63-b0aa-67f9966fae62","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:48.870Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7047a819-e8ba-4e63-b0aa-67f9966fae62","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:48.870Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7047a819-e8ba-4e63-b0aa-67f9966fae62","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:48.870Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7047a819-e8ba-4e63-b0aa-67f9966fae62","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:48.870Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7047a819-e8ba-4e63-b0aa-67f9966fae62","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:48.870Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7047a819-e8ba-4e63-b0aa-67f9966fae62","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:48.870Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7047a819-e8ba-4e63-b0aa-67f9966fae62","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:48.870Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7047a819-e8ba-4e63-b0aa-67f9966fae62","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:48.870Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7047a819-e8ba-4e63-b0aa-67f9966fae62","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:48.870Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7047a819-e8ba-4e63-b0aa-67f9966fae62","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:08:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:08:48.870Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7047a819-e8ba-4e63-b0aa-67f9966fae62","podName":"cluster-status-update-m6crj-head-9fc7t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:08:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:08:48.870Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7047a819-e8ba-4e63-b0aa-67f9966fae62","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:08:48.870Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"7047a819-e8ba-4e63-b0aa-67f9966fae62","seconds":300}
{"level":"info","ts":"2026-09-18T10:08:51.508Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"12b96192-434b-4719-9134-ffc580d77166","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:51.508Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"12b96192-434b-4719-9134-ffc580d77166","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:51.508Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"12b96192-434b-4719-9134-ffc580d77166","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:51.508Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"12b96192-434b-4719-9134-ffc580d77166"}
{"level":"info","ts":"2026-09-18T10:08:51.508Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"12b96192-434b-4719-9134-ffc580d77166","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:51.508Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"12b96192-434b-4719-9134-ffc580d77166","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:54.509Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0867a364-d4dd-40b9-8622-f0faaa5b9015","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:54.509Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0867a364-d4dd-40b9-8622-f0faaa5b9015","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:54.509Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0867a364-d4dd-40b9-8622-f0faaa5b9015","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:54.509Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0867a364-d4dd-40b9-8622-f0faaa5b9015"}
{"level":"info","ts":"2026-09-18T10:08:54.509Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0867a364-d4dd-40b9-8622-f0faaa5b9015","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:54.509Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"0867a364-d4dd-40b9-8622-f0faaa5b9015","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:56.100Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6f992477-2c73-40db-a564-e9011ee09cca"}
{"level":"info","ts":"2026-09-18T10:08:56.100Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6f992477-2c73-40db-a564-e9011ee09cca"}
{"level":"info","ts":"2026-09-18T10:08:56.100Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6f992477-2c73-40db-a564-e9011ee09cca"}
{"level":"info","ts":"2026-09-18T10:08:56.101Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6f992477-2c73-40db-a564-e9011ee09cca","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:56.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6f992477-2c73-40db-a564-e9011ee09cca","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:56.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6f992477-2c73-40db-a564-e9011ee09cca","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:56.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6f992477-2c73-40db-a564-e9011ee09cca","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:56.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6f992477-2c73-40db-a564-e9011ee09cca","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:56.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6f992477-2c73-40db-a564-e9011ee09cca","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:56.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6f992477-2c73-40db-a564-e9011ee09cca","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:56.101Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6f992477-2c73-40db-a564-e9011ee09cca","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:56.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6f992477-2c73-40db-a564-e9011ee09cca","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:56.101Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6f992477-2c73-40db-a564-e9011ee09cca","podName":"cluster-status-update-m6crj-small-group-worker-5l8bd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:56.101Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6f992477-2c73-40db-a564-e9011ee09cca","podName":"cluster-status-update-m6crj-small-group-worker-5l8bd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:08:56.101Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6f992477-2c73-40db-a564-e9011ee09cca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:08:56.101Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6f992477-2c73-40db-a564-e9011ee09cca","seconds":300}
{"level":"info","ts":"2026-09-18T10:08:57.111Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"da4d73c0-79c6-4289-a7a4-1742214f7836"}
{"level":"info","ts":"2026-09-18T10:08:57.111Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"da4d73c0-79c6-4289-a7a4-1742214f7836"}
{"level":"info","ts":"2026-09-18T10:08:57.111Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"da4d73c0-79c6-4289-a7a4-1742214f7836"}
{"level":"info","ts":"2026-09-18T10:08:57.111Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"da4d73c0-79c6-4289-a7a4-1742214f7836","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:57.111Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"da4d73c0-79c6-4289-a7a4-1742214f7836","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:57.111Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"da4d73c0-79c6-4289-a7a4-1742214f7836","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:57.111Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"da4d73c0-79c6-4289-a7a4-1742214f7836","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:57.111Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"da4d73c0-79c6-4289-a7a4-1742214f7836","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:57.111Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"da4d73c0-79c6-4289-a7a4-1742214f7836","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:57.111Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"da4d73c0-79c6-4289-a7a4-1742214f7836","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:57.111Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"da4d73c0-79c6-4289-a7a4-1742214f7836","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:57.111Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"da4d73c0-79c6-4289-a7a4-1742214f7836","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:57.111Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"da4d73c0-79c6-4289-a7a4-1742214f7836","podName":"cluster-status-update-m6crj-small-group-worker-5l8bd","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:08:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:08:57.112Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"da4d73c0-79c6-4289-a7a4-1742214f7836","podName":"cluster-status-update-m6crj-small-group-worker-5l8bd","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:08:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:08:57.112Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"da4d73c0-79c6-4289-a7a4-1742214f7836","name":"cluster-status-update-m6crj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:08:57.136Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"cbfe43a8-9b3c-463e-ab97-55fc162a3452","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:57.136Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fe0cc86a-67a6-4110-9d66-746acc0b5eab","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:57.136Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fe0cc86a-67a6-4110-9d66-746acc0b5eab","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:57.136Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fe0cc86a-67a6-4110-9d66-746acc0b5eab","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:57.136Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"0cd9621d-75c3-406e-9aee-a20b1530fd60","rayCluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:57.136Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fe0cc86a-67a6-4110-9d66-746acc0b5eab"}
{"level":"info","ts":"2026-09-18T10:08:57.136Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fe0cc86a-67a6-4110-9d66-746acc0b5eab","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:57.136Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"0cd9621d-75c3-406e-9aee-a20b1530fd60","pod":"cluster-status-update-m6crj-head-9fc7t","ip":"10.132.0.54"}
{"level":"info","ts":"2026-09-18T10:08:57.136Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"0cd9621d-75c3-406e-9aee-a20b1530fd60","pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","ip":"10.133.0.46"}
{"level":"info","ts":"2026-09-18T10:08:57.136Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fe0cc86a-67a6-4110-9d66-746acc0b5eab","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:57.136Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fe0cc86a-67a6-4110-9d66-746acc0b5eab","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:08:57.136Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"0cd9621d-75c3-406e-9aee-a20b1530fd60","rayCluster":"cluster-status-update-m6crj","podIPs":["10.132.0.54","10.133.0.46"]}
{"level":"info","ts":"2026-09-18T10:08:57.152Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"cbfe43a8-9b3c-463e-ab97-55fc162a3452","name":"cluster-status-update-m6crj-head"}
{"level":"info","ts":"2026-09-18T10:08:57.175Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"049cadf4-1af3-4f40-9ae1-8b6e852ddf29","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:57.175Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"049cadf4-1af3-4f40-9ae1-8b6e852ddf29","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:57.175Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"049cadf4-1af3-4f40-9ae1-8b6e852ddf29","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:57.175Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"049cadf4-1af3-4f40-9ae1-8b6e852ddf29"}
{"level":"info","ts":"2026-09-18T10:08:57.175Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"049cadf4-1af3-4f40-9ae1-8b6e852ddf29","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:57.175Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"049cadf4-1af3-4f40-9ae1-8b6e852ddf29","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:57.182Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"cbfe43a8-9b3c-463e-ab97-55fc162a3452","name":"cluster-status-update-m6crj-workers"}
{"level":"info","ts":"2026-09-18T10:08:57.182Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"cbfe43a8-9b3c-463e-ab97-55fc162a3452","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:57.510Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a9381c07-a467-4cf8-8b17-bb1e01b87aac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:57.510Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a9381c07-a467-4cf8-8b17-bb1e01b87aac","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:57.510Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a9381c07-a467-4cf8-8b17-bb1e01b87aac","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:57.510Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a9381c07-a467-4cf8-8b17-bb1e01b87aac"}
{"level":"info","ts":"2026-09-18T10:08:57.510Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a9381c07-a467-4cf8-8b17-bb1e01b87aac","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:57.510Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a9381c07-a467-4cf8-8b17-bb1e01b87aac","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:57.880Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6c46330d-87e3-4eea-8c4c-f4a7fdb26d38"}
{"level":"info","ts":"2026-09-18T10:08:57.880Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6c46330d-87e3-4eea-8c4c-f4a7fdb26d38"}
{"level":"info","ts":"2026-09-18T10:08:57.880Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6c46330d-87e3-4eea-8c4c-f4a7fdb26d38"}
{"level":"info","ts":"2026-09-18T10:08:57.880Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6c46330d-87e3-4eea-8c4c-f4a7fdb26d38","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:57.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6c46330d-87e3-4eea-8c4c-f4a7fdb26d38","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:57.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6c46330d-87e3-4eea-8c4c-f4a7fdb26d38","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:57.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6c46330d-87e3-4eea-8c4c-f4a7fdb26d38","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:57.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6c46330d-87e3-4eea-8c4c-f4a7fdb26d38","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:57.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6c46330d-87e3-4eea-8c4c-f4a7fdb26d38","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:57.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6c46330d-87e3-4eea-8c4c-f4a7fdb26d38","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:57.880Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6c46330d-87e3-4eea-8c4c-f4a7fdb26d38","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:57.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6c46330d-87e3-4eea-8c4c-f4a7fdb26d38","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:57.880Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6c46330d-87e3-4eea-8c4c-f4a7fdb26d38","podName":"cluster-status-update-m6crj-small-group-worker-5l8bd","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:08:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:08:57.880Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6c46330d-87e3-4eea-8c4c-f4a7fdb26d38","podName":"cluster-status-update-m6crj-small-group-worker-5l8bd","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:08:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:08:57.880Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"6c46330d-87e3-4eea-8c4c-f4a7fdb26d38","name":"cluster-status-update-m6crj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:08:57.901Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"22955c64-bde4-4b29-a2ab-a557b30268f6","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:57.901Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"e64b8a06-1b26-4c51-9729-961aaf4af226","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:57.901Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"e64b8a06-1b26-4c51-9729-961aaf4af226","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:57.901Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"e64b8a06-1b26-4c51-9729-961aaf4af226","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:57.901Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"6dda6069-da89-45de-9619-ab96bbb3ce6b","rayCluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:57.901Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"6dda6069-da89-45de-9619-ab96bbb3ce6b","pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","ip":"10.133.0.46"}
{"level":"info","ts":"2026-09-18T10:08:57.901Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"6dda6069-da89-45de-9619-ab96bbb3ce6b","pod":"cluster-status-update-m6crj-head-9fc7t","ip":"10.132.0.54"}
{"level":"info","ts":"2026-09-18T10:08:57.901Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"6dda6069-da89-45de-9619-ab96bbb3ce6b","rayCluster":"cluster-status-update-m6crj","podIPs":["10.133.0.46","10.132.0.54"]}
{"level":"info","ts":"2026-09-18T10:08:57.901Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"e64b8a06-1b26-4c51-9729-961aaf4af226"}
{"level":"info","ts":"2026-09-18T10:08:57.901Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"e64b8a06-1b26-4c51-9729-961aaf4af226","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:57.901Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"e64b8a06-1b26-4c51-9729-961aaf4af226","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:57.901Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"e64b8a06-1b26-4c51-9729-961aaf4af226","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:08:57.917Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"22955c64-bde4-4b29-a2ab-a557b30268f6","name":"cluster-status-update-m6crj-head"}
{"level":"info","ts":"2026-09-18T10:08:57.919Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fe7d9e27-db1d-429a-8130-e8fbe5ed89f3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:08:57.919Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fe7d9e27-db1d-429a-8130-e8fbe5ed89f3","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:57.919Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fe7d9e27-db1d-429a-8130-e8fbe5ed89f3","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:08:57.919Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fe7d9e27-db1d-429a-8130-e8fbe5ed89f3"}
{"level":"info","ts":"2026-09-18T10:08:57.919Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fe7d9e27-db1d-429a-8130-e8fbe5ed89f3","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:08:57.919Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"fe7d9e27-db1d-429a-8130-e8fbe5ed89f3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:08:57.931Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"22955c64-bde4-4b29-a2ab-a557b30268f6","name":"cluster-status-update-m6crj-workers"}
{"level":"info","ts":"2026-09-18T10:08:57.931Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"22955c64-bde4-4b29-a2ab-a557b30268f6","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:08:59.138Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2dfcb612-1a82-49d5-b016-a3b303242470"}
{"level":"info","ts":"2026-09-18T10:08:59.138Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2dfcb612-1a82-49d5-b016-a3b303242470"}
{"level":"info","ts":"2026-09-18T10:08:59.138Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2dfcb612-1a82-49d5-b016-a3b303242470"}
{"level":"info","ts":"2026-09-18T10:08:59.138Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2dfcb612-1a82-49d5-b016-a3b303242470","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:08:59.138Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2dfcb612-1a82-49d5-b016-a3b303242470","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:08:59.138Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2dfcb612-1a82-49d5-b016-a3b303242470","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:59.138Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2dfcb612-1a82-49d5-b016-a3b303242470","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:08:59.138Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2dfcb612-1a82-49d5-b016-a3b303242470","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:08:59.138Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2dfcb612-1a82-49d5-b016-a3b303242470","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:59.138Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2dfcb612-1a82-49d5-b016-a3b303242470","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:08:59.138Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2dfcb612-1a82-49d5-b016-a3b303242470","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:08:59.138Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2dfcb612-1a82-49d5-b016-a3b303242470","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:08:59.138Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2dfcb612-1a82-49d5-b016-a3b303242470","podName":"cluster-status-update-m6crj-small-group-worker-5l8bd","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:08:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:08:59.138Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2dfcb612-1a82-49d5-b016-a3b303242470","podName":"cluster-status-update-m6crj-small-group-worker-5l8bd","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:08:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:08:59.138Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2dfcb612-1a82-49d5-b016-a3b303242470","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:08:59.138Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"2dfcb612-1a82-49d5-b016-a3b303242470","seconds":300}
{"level":"info","ts":"2026-09-18T10:09:00.511Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"c925e112-b3f6-4b54-be9e-f718e0699569","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:00.511Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"c925e112-b3f6-4b54-be9e-f718e0699569","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:00.511Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"c925e112-b3f6-4b54-be9e-f718e0699569","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:00.511Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"c925e112-b3f6-4b54-be9e-f718e0699569"}
{"level":"info","ts":"2026-09-18T10:09:00.511Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"c925e112-b3f6-4b54-be9e-f718e0699569","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:09:00.511Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"c925e112-b3f6-4b54-be9e-f718e0699569","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:03.513Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"27e2c27b-0b57-4150-814a-813967442b95","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:03.513Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"27e2c27b-0b57-4150-814a-813967442b95","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:03.513Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"27e2c27b-0b57-4150-814a-813967442b95","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:03.513Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"27e2c27b-0b57-4150-814a-813967442b95"}
{"level":"info","ts":"2026-09-18T10:09:03.513Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"27e2c27b-0b57-4150-814a-813967442b95","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:09:03.513Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"27e2c27b-0b57-4150-814a-813967442b95","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:06.513Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"b724bf3d-11d0-49b7-a8cc-5617f454001f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:06.513Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"b724bf3d-11d0-49b7-a8cc-5617f454001f","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:06.513Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"b724bf3d-11d0-49b7-a8cc-5617f454001f","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:06.513Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"b724bf3d-11d0-49b7-a8cc-5617f454001f"}
{"level":"info","ts":"2026-09-18T10:09:06.513Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"b724bf3d-11d0-49b7-a8cc-5617f454001f","RayCluster":"cluster-status-update-m6crj","State":""}
{"level":"info","ts":"2026-09-18T10:09:06.513Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"b724bf3d-11d0-49b7-a8cc-5617f454001f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:06.793Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"135409b6-3a7c-4d77-9803-2de85536019d","rayCluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:06.793Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"135409b6-3a7c-4d77-9803-2de85536019d","pod":"cluster-status-update-m6crj-head-9fc7t","ip":"10.132.0.54"}
{"level":"info","ts":"2026-09-18T10:09:06.793Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"135409b6-3a7c-4d77-9803-2de85536019d","pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","ip":"10.133.0.46"}
{"level":"info","ts":"2026-09-18T10:09:06.793Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"135409b6-3a7c-4d77-9803-2de85536019d","rayCluster":"cluster-status-update-m6crj","podIPs":["10.132.0.54","10.133.0.46"]}
{"level":"info","ts":"2026-09-18T10:09:08.132Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"f56b99e2-8380-4230-8d5b-33f9bf12ea6a"}
{"level":"info","ts":"2026-09-18T10:09:08.132Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"f56b99e2-8380-4230-8d5b-33f9bf12ea6a"}
{"level":"info","ts":"2026-09-18T10:09:08.132Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"f56b99e2-8380-4230-8d5b-33f9bf12ea6a"}
{"level":"info","ts":"2026-09-18T10:09:08.132Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"f56b99e2-8380-4230-8d5b-33f9bf12ea6a","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:08.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"f56b99e2-8380-4230-8d5b-33f9bf12ea6a","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:08.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"f56b99e2-8380-4230-8d5b-33f9bf12ea6a","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:08.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"f56b99e2-8380-4230-8d5b-33f9bf12ea6a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:08.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"f56b99e2-8380-4230-8d5b-33f9bf12ea6a","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:08.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"f56b99e2-8380-4230-8d5b-33f9bf12ea6a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:08.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"f56b99e2-8380-4230-8d5b-33f9bf12ea6a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:08.132Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"f56b99e2-8380-4230-8d5b-33f9bf12ea6a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:08.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"f56b99e2-8380-4230-8d5b-33f9bf12ea6a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:08.132Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"f56b99e2-8380-4230-8d5b-33f9bf12ea6a","name":"cluster-status-update-m6crj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:09:08Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:09:08Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:09:08.151Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"a2491e5b-0add-476d-b386-21870f36cd65","rayCluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:08.151Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"89670563-bc96-4d0c-bdcd-6c5ce22deae0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:08.151Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"89670563-bc96-4d0c-bdcd-6c5ce22deae0","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:08.151Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"89670563-bc96-4d0c-bdcd-6c5ce22deae0","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:08.151Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"a2491e5b-0add-476d-b386-21870f36cd65","pod":"cluster-status-update-m6crj-head-9fc7t","ip":"10.132.0.54"}
{"level":"info","ts":"2026-09-18T10:09:08.151Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"a2491e5b-0add-476d-b386-21870f36cd65","pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","ip":"10.133.0.46"}
{"level":"info","ts":"2026-09-18T10:09:08.151Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"a2491e5b-0add-476d-b386-21870f36cd65","rayCluster":"cluster-status-update-m6crj","podIPs":["10.132.0.54","10.133.0.46"]}
{"level":"info","ts":"2026-09-18T10:09:08.151Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"89670563-bc96-4d0c-bdcd-6c5ce22deae0"}
{"level":"info","ts":"2026-09-18T10:09:08.151Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"c439a750-f2c8-44ab-be84-f2f6e910f6fa","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:08.178Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"89670563-bc96-4d0c-bdcd-6c5ce22deae0","Kubernetes Job":"cluster-status-update"}
{"level":"info","ts":"2026-09-18T10:09:08.178Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"89670563-bc96-4d0c-bdcd-6c5ce22deae0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","dashboardURL":"cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:08.178Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"89670563-bc96-4d0c-bdcd-6c5ce22deae0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:09:08.190Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"c439a750-f2c8-44ab-be84-f2f6e910f6fa","name":"cluster-status-update-m6crj-head"}
{"level":"info","ts":"2026-09-18T10:09:08.197Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"dea78a2d-813b-4940-bc2e-fa90323b3f7d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:08.197Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"dea78a2d-813b-4940-bc2e-fa90323b3f7d","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:08.197Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"dea78a2d-813b-4940-bc2e-fa90323b3f7d","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:08.197Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"dea78a2d-813b-4940-bc2e-fa90323b3f7d"}
{"level":"info","ts":"2026-09-18T10:09:08.197Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"dea78a2d-813b-4940-bc2e-fa90323b3f7d","Kubernetes Job":"cluster-status-update"}
{"level":"info","ts":"2026-09-18T10:09:08.197Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"dea78a2d-813b-4940-bc2e-fa90323b3f7d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","dashboardURL":"cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:08.197Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"dea78a2d-813b-4940-bc2e-fa90323b3f7d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:09:08.218Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"dea78a2d-813b-4940-bc2e-fa90323b3f7d","error":"Operation cannot be fulfilled on rayjobs.ray.io \"cluster-status-update\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T10:09:08.218Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"dea78a2d-813b-4940-bc2e-fa90323b3f7d"}
{"level":"error","ts":"2026-09-18T10:09:08.218Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"dea78a2d-813b-4940-bc2e-fa90323b3f7d","error":"Operation cannot be fulfilled on rayjobs.ray.io \"cluster-status-update\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:08.218Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"690c54f5-935a-4bd5-9004-1ed56fa9db05","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:08.218Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"690c54f5-935a-4bd5-9004-1ed56fa9db05","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:08.218Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"690c54f5-935a-4bd5-9004-1ed56fa9db05","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:08.219Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"690c54f5-935a-4bd5-9004-1ed56fa9db05"}
{"level":"info","ts":"2026-09-18T10:09:08.219Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"690c54f5-935a-4bd5-9004-1ed56fa9db05","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:09:08.243Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"c439a750-f2c8-44ab-be84-f2f6e910f6fa","name":"cluster-status-update-m6crj-workers"}
{"level":"info","ts":"2026-09-18T10:09:08.243Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"c439a750-f2c8-44ab-be84-f2f6e910f6fa","cluster":"cluster-status-update-m6crj"}
{"level":"error","ts":"2026-09-18T10:09:08.255Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"690c54f5-935a-4bd5-9004-1ed56fa9db05","JobId":"cluster-status-update-j2fcv","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:08.255Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"690c54f5-935a-4bd5-9004-1ed56fa9db05"}
{"level":"error","ts":"2026-09-18T10:09:08.255Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"690c54f5-935a-4bd5-9004-1ed56fa9db05","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:08.255Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5a21def7-ea51-4762-b622-4bce3ffca8da","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:08.255Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5a21def7-ea51-4762-b622-4bce3ffca8da","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:08.255Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5a21def7-ea51-4762-b622-4bce3ffca8da","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:08.256Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5a21def7-ea51-4762-b622-4bce3ffca8da"}
{"level":"info","ts":"2026-09-18T10:09:08.256Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5a21def7-ea51-4762-b622-4bce3ffca8da","rayCluster":"cluster-status-update-m6crj","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:09:08.260Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5a21def7-ea51-4762-b622-4bce3ffca8da","JobId":"cluster-status-update-j2fcv","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:08.260Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5a21def7-ea51-4762-b622-4bce3ffca8da"}
{"level":"error","ts":"2026-09-18T10:09:08.260Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5a21def7-ea51-4762-b622-4bce3ffca8da","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:08.266Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5eb5d7dd-1173-45d5-8ba9-5eb64b45a398","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:08.266Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5eb5d7dd-1173-45d5-8ba9-5eb64b45a398","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:08.266Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5eb5d7dd-1173-45d5-8ba9-5eb64b45a398","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:08.266Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5eb5d7dd-1173-45d5-8ba9-5eb64b45a398"}
{"level":"info","ts":"2026-09-18T10:09:08.266Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5eb5d7dd-1173-45d5-8ba9-5eb64b45a398","rayCluster":"cluster-status-update-m6crj","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:09:08.270Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5eb5d7dd-1173-45d5-8ba9-5eb64b45a398","JobId":"cluster-status-update-j2fcv","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:08.270Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5eb5d7dd-1173-45d5-8ba9-5eb64b45a398"}
{"level":"error","ts":"2026-09-18T10:09:08.270Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5eb5d7dd-1173-45d5-8ba9-5eb64b45a398","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:08.311Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"199c09ab-1ef6-4f76-b40f-60e4193942d3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:08.311Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"199c09ab-1ef6-4f76-b40f-60e4193942d3","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:08.311Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"199c09ab-1ef6-4f76-b40f-60e4193942d3","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:08.311Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"199c09ab-1ef6-4f76-b40f-60e4193942d3"}
{"level":"info","ts":"2026-09-18T10:09:08.311Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"199c09ab-1ef6-4f76-b40f-60e4193942d3","rayCluster":"cluster-status-update-m6crj","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:09:08.315Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"199c09ab-1ef6-4f76-b40f-60e4193942d3","JobId":"cluster-status-update-j2fcv","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:08.315Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"199c09ab-1ef6-4f76-b40f-60e4193942d3"}
{"level":"error","ts":"2026-09-18T10:09:08.315Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"199c09ab-1ef6-4f76-b40f-60e4193942d3","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:08.396Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"93e1bda6-2db1-4a16-b373-070151cbae74","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:08.396Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"93e1bda6-2db1-4a16-b373-070151cbae74","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:08.396Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"93e1bda6-2db1-4a16-b373-070151cbae74","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:08.396Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"93e1bda6-2db1-4a16-b373-070151cbae74"}
{"level":"info","ts":"2026-09-18T10:09:08.396Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"93e1bda6-2db1-4a16-b373-070151cbae74","rayCluster":"cluster-status-update-m6crj","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:09:08.401Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"93e1bda6-2db1-4a16-b373-070151cbae74","JobId":"cluster-status-update-j2fcv","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:08.401Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"93e1bda6-2db1-4a16-b373-070151cbae74"}
{"level":"error","ts":"2026-09-18T10:09:08.401Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"93e1bda6-2db1-4a16-b373-070151cbae74","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:08.562Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"9cb3ca6b-abbc-443e-82fe-edd3a62d13b1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:08.562Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"9cb3ca6b-abbc-443e-82fe-edd3a62d13b1","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:08.562Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"9cb3ca6b-abbc-443e-82fe-edd3a62d13b1","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:08.562Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"9cb3ca6b-abbc-443e-82fe-edd3a62d13b1"}
{"level":"info","ts":"2026-09-18T10:09:08.562Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"9cb3ca6b-abbc-443e-82fe-edd3a62d13b1","rayCluster":"cluster-status-update-m6crj","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:09:08.567Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"9cb3ca6b-abbc-443e-82fe-edd3a62d13b1","JobId":"cluster-status-update-j2fcv","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:08.568Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"9cb3ca6b-abbc-443e-82fe-edd3a62d13b1"}
{"level":"error","ts":"2026-09-18T10:09:08.568Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"9cb3ca6b-abbc-443e-82fe-edd3a62d13b1","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:08.888Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"07eeab0a-f642-4ba6-942b-60912a480f5d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:08.888Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"07eeab0a-f642-4ba6-942b-60912a480f5d","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:08.888Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"07eeab0a-f642-4ba6-942b-60912a480f5d","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:08.888Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"07eeab0a-f642-4ba6-942b-60912a480f5d"}
{"level":"info","ts":"2026-09-18T10:09:08.888Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"07eeab0a-f642-4ba6-942b-60912a480f5d","rayCluster":"cluster-status-update-m6crj","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:09:08.894Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"07eeab0a-f642-4ba6-942b-60912a480f5d","JobId":"cluster-status-update-j2fcv","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:08.894Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"07eeab0a-f642-4ba6-942b-60912a480f5d"}
{"level":"error","ts":"2026-09-18T10:09:08.894Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"07eeab0a-f642-4ba6-942b-60912a480f5d","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:09.208Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"ac8cd964-c39e-480d-954d-7479c9351efa","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:09.208Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"ac8cd964-c39e-480d-954d-7479c9351efa","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:09.208Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"ac8cd964-c39e-480d-954d-7479c9351efa","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:09.208Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"ac8cd964-c39e-480d-954d-7479c9351efa"}
{"level":"info","ts":"2026-09-18T10:09:09.208Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"ac8cd964-c39e-480d-954d-7479c9351efa","rayCluster":"cluster-status-update-m6crj","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:09:09.214Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"ac8cd964-c39e-480d-954d-7479c9351efa","JobId":"cluster-status-update-j2fcv","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:09.214Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"ac8cd964-c39e-480d-954d-7479c9351efa"}
{"level":"error","ts":"2026-09-18T10:09:09.214Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"ac8cd964-c39e-480d-954d-7479c9351efa","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:09.535Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"be6ff372-c098-4ca3-b99d-aa6236979e7f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:09.535Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"be6ff372-c098-4ca3-b99d-aa6236979e7f","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:09.535Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"be6ff372-c098-4ca3-b99d-aa6236979e7f","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:09.535Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"be6ff372-c098-4ca3-b99d-aa6236979e7f"}
{"level":"info","ts":"2026-09-18T10:09:09.535Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"be6ff372-c098-4ca3-b99d-aa6236979e7f","rayCluster":"cluster-status-update-m6crj","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:09:09.540Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"be6ff372-c098-4ca3-b99d-aa6236979e7f","JobId":"cluster-status-update-j2fcv","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:09.541Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"be6ff372-c098-4ca3-b99d-aa6236979e7f"}
{"level":"error","ts":"2026-09-18T10:09:09.541Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"be6ff372-c098-4ca3-b99d-aa6236979e7f","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:10.156Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cf522c93-76a1-4739-97f4-29ef7d5775db"}
{"level":"info","ts":"2026-09-18T10:09:10.156Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cf522c93-76a1-4739-97f4-29ef7d5775db"}
{"level":"info","ts":"2026-09-18T10:09:10.156Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cf522c93-76a1-4739-97f4-29ef7d5775db"}
{"level":"info","ts":"2026-09-18T10:09:10.156Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cf522c93-76a1-4739-97f4-29ef7d5775db","1 head service found":"cluster-status-update-m6crj-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:10.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cf522c93-76a1-4739-97f4-29ef7d5775db","Found 1 head Pod":"cluster-status-update-m6crj-head-9fc7t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:10.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cf522c93-76a1-4739-97f4-29ef7d5775db","head Pod":"cluster-status-update-m6crj-head-9fc7t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-m6crj-head-9fc7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:10.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cf522c93-76a1-4739-97f4-29ef7d5775db","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:10.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cf522c93-76a1-4739-97f4-29ef7d5775db","worker Pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-m6crj-small-group-worker-5l8bd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:10.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cf522c93-76a1-4739-97f4-29ef7d5775db","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:10.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cf522c93-76a1-4739-97f4-29ef7d5775db","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:10.156Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cf522c93-76a1-4739-97f4-29ef7d5775db","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:10.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cf522c93-76a1-4739-97f4-29ef7d5775db","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:10.156Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cf522c93-76a1-4739-97f4-29ef7d5775db","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:09:10.156Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"cf522c93-76a1-4739-97f4-29ef7d5775db","seconds":300}
{"level":"info","ts":"2026-09-18T10:09:12.101Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a66649d6-e621-4cad-b45c-6f4206390763","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:12.101Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a66649d6-e621-4cad-b45c-6f4206390763","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:12.101Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a66649d6-e621-4cad-b45c-6f4206390763","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:12.101Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a66649d6-e621-4cad-b45c-6f4206390763"}
{"level":"info","ts":"2026-09-18T10:09:12.102Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a66649d6-e621-4cad-b45c-6f4206390763","rayCluster":"cluster-status-update-m6crj","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:09:12.106Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a66649d6-e621-4cad-b45c-6f4206390763","JobId":"cluster-status-update-j2fcv","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:12.107Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a66649d6-e621-4cad-b45c-6f4206390763"}
{"level":"error","ts":"2026-09-18T10:09:12.107Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"a66649d6-e621-4cad-b45c-6f4206390763","error":"Job cluster-status-update-j2fcv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:17.227Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5f16a102-0da5-4f20-a13d-fe0bd706598b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:17.227Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5f16a102-0da5-4f20-a13d-fe0bd706598b","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:17.227Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5f16a102-0da5-4f20-a13d-fe0bd706598b","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:17.227Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5f16a102-0da5-4f20-a13d-fe0bd706598b"}
{"level":"info","ts":"2026-09-18T10:09:17.227Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5f16a102-0da5-4f20-a13d-fe0bd706598b","rayCluster":"cluster-status-update-m6crj","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:09:17.233Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5f16a102-0da5-4f20-a13d-fe0bd706598b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","dashboardURL":"cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:08:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:09:16Z"},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","dashboardURL":"cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:09:08Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:09:08Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:17.233Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5f16a102-0da5-4f20-a13d-fe0bd706598b","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:09:17.259Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"4d6c99cd-c9b7-4252-8646-8570ba54af4a","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:17.259Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"4d6c99cd-c9b7-4252-8646-8570ba54af4a","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:17.259Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"4d6c99cd-c9b7-4252-8646-8570ba54af4a","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:17.259Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"4d6c99cd-c9b7-4252-8646-8570ba54af4a"}
{"level":"info","ts":"2026-09-18T10:09:17.259Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"4d6c99cd-c9b7-4252-8646-8570ba54af4a","rayCluster":"cluster-status-update-m6crj","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:09:17.262Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"4d6c99cd-c9b7-4252-8646-8570ba54af4a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:09:16Z"},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","dashboardURL":"cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:09:08Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:09:08Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:09:16Z"},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","dashboardURL":"cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:09:08Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:09:08Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:20.259Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"70ae0787-6825-4c0c-9fc9-b6bcb475e59a","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:20.259Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"70ae0787-6825-4c0c-9fc9-b6bcb475e59a","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:20.259Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"70ae0787-6825-4c0c-9fc9-b6bcb475e59a","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:20.259Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"70ae0787-6825-4c0c-9fc9-b6bcb475e59a"}
{"level":"info","ts":"2026-09-18T10:09:20.259Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"70ae0787-6825-4c0c-9fc9-b6bcb475e59a","rayCluster":"cluster-status-update-m6crj","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:09:20.262Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"70ae0787-6825-4c0c-9fc9-b6bcb475e59a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:09:16Z"},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","dashboardURL":"cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:09:08Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:09:08Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:09:16Z"},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","dashboardURL":"cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:09:08Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:09:08Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:20.262Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"70ae0787-6825-4c0c-9fc9-b6bcb475e59a","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:09:20.284Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"ba402ac3-bd1c-4ad5-8813-289803bca89e","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:20.284Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"ba402ac3-bd1c-4ad5-8813-289803bca89e","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:20.284Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"ba402ac3-bd1c-4ad5-8813-289803bca89e","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:20.284Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"ba402ac3-bd1c-4ad5-8813-289803bca89e"}
{"level":"info","ts":"2026-09-18T10:09:20.284Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"ba402ac3-bd1c-4ad5-8813-289803bca89e","rayCluster":"cluster-status-update-m6crj","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:09:20.287Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"ba402ac3-bd1c-4ad5-8813-289803bca89e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:09:16Z"},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","dashboardURL":"cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:09:08Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:09:08Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:09:16Z"},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","dashboardURL":"cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:09:08Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:09:08Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:20.835Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"1d4bbdd7-8c36-44c7-8290-f06cd870f2e2","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:20.835Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"9cb1d4d0-8fc7-431c-989d-241ab9aa887d","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:20.835Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"9cb1d4d0-8fc7-431c-989d-241ab9aa887d","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:20.835Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"9cb1d4d0-8fc7-431c-989d-241ab9aa887d","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:20.836Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"9cb1d4d0-8fc7-431c-989d-241ab9aa887d"}
{"level":"info","ts":"2026-09-18T10:09:20.836Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"9cb1d4d0-8fc7-431c-989d-241ab9aa887d","rayCluster":"cluster-status-update-m6crj","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:09:20.836Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"28e8533f-7efb-420f-ae32-ad76b3d6a4ed","rayCluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:20.836Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"28e8533f-7efb-420f-ae32-ad76b3d6a4ed","pod":"cluster-status-update-m6crj-head-9fc7t","ip":"10.132.0.54"}
{"level":"info","ts":"2026-09-18T10:09:20.836Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"28e8533f-7efb-420f-ae32-ad76b3d6a4ed","pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","ip":"10.133.0.46"}
{"level":"info","ts":"2026-09-18T10:09:20.836Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"28e8533f-7efb-420f-ae32-ad76b3d6a4ed","rayCluster":"cluster-status-update-m6crj","podIPs":["10.132.0.54","10.133.0.46"]}
{"level":"info","ts":"2026-09-18T10:09:20.839Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"9cb1d4d0-8fc7-431c-989d-241ab9aa887d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:09:16Z"},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","dashboardURL":"cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:09:08Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:09:08Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:09:16Z"},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","dashboardURL":"cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:09:08Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:09:08Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:20.839Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"9cb1d4d0-8fc7-431c-989d-241ab9aa887d","old JobStatus":"RUNNING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:09:20.854Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"1d4bbdd7-8c36-44c7-8290-f06cd870f2e2","name":"cluster-status-update-m6crj-head"}
{"level":"info","ts":"2026-09-18T10:09:20.856Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"375f3df3-cbae-44a6-b64d-5be7f0623275","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:20.856Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"375f3df3-cbae-44a6-b64d-5be7f0623275","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:20.856Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"375f3df3-cbae-44a6-b64d-5be7f0623275","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:20.856Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"375f3df3-cbae-44a6-b64d-5be7f0623275"}
{"level":"info","ts":"2026-09-18T10:09:20.856Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"375f3df3-cbae-44a6-b64d-5be7f0623275","rayCluster":"cluster-status-update-m6crj","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:09:20.859Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"375f3df3-cbae-44a6-b64d-5be7f0623275","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:09:16Z"},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","dashboardURL":"cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:09:08Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:09:08Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:09:16Z"},"jobId":"cluster-status-update-j2fcv","rayClusterName":"cluster-status-update-m6crj","dashboardURL":"cluster-status-update-m6crj-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-09-18T10:08:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:09:08Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"cluster-status-update-m6crj-head-9fc7t","serviceName":"cluster-status-update-m6crj-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:08:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:08:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:09:08Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:08:36Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:20.870Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"1d4bbdd7-8c36-44c7-8290-f06cd870f2e2","name":"cluster-status-update-m6crj-workers"}
{"level":"info","ts":"2026-09-18T10:09:20.870Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"1d4bbdd7-8c36-44c7-8290-f06cd870f2e2","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:21.874Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5f73540f-6c29-4347-9ed4-ae11bbeef697","DeletionTimestamp":"2026-09-18 10:09:21 +0000 UTC"}
{"level":"info","ts":"2026-09-18T10:09:21.882Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"5f73540f-6c29-4347-9ed4-ae11bbeef697","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T10:09:21.900Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c56a999a-84ae-47f9-8039-7511854b79f7","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:21.900Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c56a999a-84ae-47f9-8039-7511854b79f7","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T10:09:21.915Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c56a999a-84ae-47f9-8039-7511854b79f7"}
{"level":"info","ts":"2026-09-18T10:09:21.915Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c56a999a-84ae-47f9-8039-7511854b79f7","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T10:09:21.915Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c56a999a-84ae-47f9-8039-7511854b79f7","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:09:21.915Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c56a999a-84ae-47f9-8039-7511854b79f7","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:09:21.930Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"4b818c12-91a0-4f18-9768-f6d4bd6c4ef1","rayCluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:21.930Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"0d40feb0-fa7a-442c-b330-ee1e1be513e2"}
{"level":"info","ts":"2026-09-18T10:09:21.930Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"db1d05f9-4654-446d-88bd-8ae7c2a30f25","namespacedName":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:21.930Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"4b818c12-91a0-4f18-9768-f6d4bd6c4ef1","pod":"cluster-status-update-m6crj-head-9fc7t","ip":"10.132.0.54"}
{"level":"info","ts":"2026-09-18T10:09:21.930Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"4b818c12-91a0-4f18-9768-f6d4bd6c4ef1","pod":"cluster-status-update-m6crj-small-group-worker-5l8bd","ip":"10.133.0.46"}
{"level":"info","ts":"2026-09-18T10:09:21.930Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"db1d05f9-4654-446d-88bd-8ae7c2a30f25","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:21.930Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"db1d05f9-4654-446d-88bd-8ae7c2a30f25","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:21.930Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"4b818c12-91a0-4f18-9768-f6d4bd6c4ef1","rayCluster":"cluster-status-update-m6crj","podIPs":["10.132.0.54","10.133.0.46"]}
{"level":"info","ts":"2026-09-18T10:09:21.930Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"db1d05f9-4654-446d-88bd-8ae7c2a30f25","namespace":"test-ns-ltx56","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-09-18T10:09:21.930Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"db1d05f9-4654-446d-88bd-8ae7c2a30f25","name":"kuberay-gateway-access","namespace":"test-ns-ltx56","remainingClusters":1}
{"level":"info","ts":"2026-09-18T10:09:21.930Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"reconcileID":"0a5e1170-d0f9-4ac1-8501-5b6be412efc2"}
{"level":"info","ts":"2026-09-18T10:09:21.931Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"1300329d-673f-4a1a-bbe1-52debd0944bd"}
{"level":"info","ts":"2026-09-18T10:09:21.931Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"dec7c7e7-0157-4b82-adc5-5a86e1a9398e","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:21.931Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"dec7c7e7-0157-4b82-adc5-5a86e1a9398e"}
{"level":"info","ts":"2026-09-18T10:09:21.931Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"dec7c7e7-0157-4b82-adc5-5a86e1a9398e","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T10:09:21.931Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"dec7c7e7-0157-4b82-adc5-5a86e1a9398e","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-6lgq2","rayClusterName":"counter-auth-9sxwh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:09:21.931Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"dec7c7e7-0157-4b82-adc5-5a86e1a9398e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:09:21.940Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"db1d05f9-4654-446d-88bd-8ae7c2a30f25","name":"test-ns-ltx56-cluster-status-update-m6crj","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T10:09:21.947Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"dec7c7e7-0157-4b82-adc5-5a86e1a9398e","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter-auth\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T10:09:21.947Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"dec7c7e7-0157-4b82-adc5-5a86e1a9398e"}
{"level":"error","ts":"2026-09-18T10:09:21.947Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"dec7c7e7-0157-4b82-adc5-5a86e1a9398e","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:21.948Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a51a7e76-08f2-4bc3-84c4-c989b11279db","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:21.948Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a51a7e76-08f2-4bc3-84c4-c989b11279db","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:21.948Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a51a7e76-08f2-4bc3-84c4-c989b11279db","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:21.948Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a51a7e76-08f2-4bc3-84c4-c989b11279db","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:21.954Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"db1d05f9-4654-446d-88bd-8ae7c2a30f25","configMap":"kube-rbac-proxy-config-cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:21.954Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:21.955Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:09:21.967Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"db1d05f9-4654-446d-88bd-8ae7c2a30f25","serviceAccount":"cluster-status-update-m6crj-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:09:21.969Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"cd1c14b1-df10-4da1-9f51-1688d6f061a9","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:21.969Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6ee87b74-855a-4b2d-9c5a-15ab383b0e4a"}
{"level":"info","ts":"2026-09-18T10:09:21.969Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6ee87b74-855a-4b2d-9c5a-15ab383b0e4a"}
{"level":"info","ts":"2026-09-18T10:09:21.969Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"cd1c14b1-df10-4da1-9f51-1688d6f061a9","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:21.969Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6ee87b74-855a-4b2d-9c5a-15ab383b0e4a"}
{"level":"info","ts":"2026-09-18T10:09:21.969Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"80fc35d4-bcf5-43ed-8cc8-e60b737b8d95","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:21.970Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a51a7e76-08f2-4bc3-84c4-c989b11279db","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:21.970Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a51a7e76-08f2-4bc3-84c4-c989b11279db"}
{"level":"info","ts":"2026-09-18T10:09:21.970Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a51a7e76-08f2-4bc3-84c4-c989b11279db","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:21.970Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a51a7e76-08f2-4bc3-84c4-c989b11279db","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:09:21.971Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"12fca2c9-c3bf-42dc-88bc-b58fed739275","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:21.971Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"12fca2c9-c3bf-42dc-88bc-b58fed739275","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:21.971Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"12fca2c9-c3bf-42dc-88bc-b58fed739275","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:21.971Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"12fca2c9-c3bf-42dc-88bc-b58fed739275"}
{"level":"info","ts":"2026-09-18T10:09:21.971Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"12fca2c9-c3bf-42dc-88bc-b58fed739275","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:21.971Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"12fca2c9-c3bf-42dc-88bc-b58fed739275","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:09:21.980Z","logger":"raycluster-resource","msg":"default","name":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:21.980Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:09:21.994Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6ee87b74-855a-4b2d-9c5a-15ab383b0e4a","name":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:21.994Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"cd1c14b1-df10-4da1-9f51-1688d6f061a9","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:21.994Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"cd1c14b1-df10-4da1-9f51-1688d6f061a9","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:21.994Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6ee87b74-855a-4b2d-9c5a-15ab383b0e4a","error":"head certificate secret ray-head-secret-counter-auth-hw6hh not found"}
{"level":"error","ts":"2026-09-18T10:09:21.994Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6ee87b74-855a-4b2d-9c5a-15ab383b0e4a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:21.994Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6ee87b74-855a-4b2d-9c5a-15ab383b0e4a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found"}
{"level":"info","ts":"2026-09-18T10:09:21.994Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6ee87b74-855a-4b2d-9c5a-15ab383b0e4a","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:21.994Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6ee87b74-855a-4b2d-9c5a-15ab383b0e4a","serviceSelectors":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/identifier":"counter-auth-hw6hh-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:21.994Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6ee87b74-855a-4b2d-9c5a-15ab383b0e4a","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:21.994Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6ee87b74-855a-4b2d-9c5a-15ab383b0e4a","error":"unable to find head service. cluster name counter-auth-hw6hh, filter labels [test-ns-ltx56 map[ray.io/cluster:counter-auth-hw6hh ray.io/identifier:counter-auth-hw6hh-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-09-18T10:09:21.994Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6ee87b74-855a-4b2d-9c5a-15ab383b0e4a"}
{"level":"error","ts":"2026-09-18T10:09:21.994Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6ee87b74-855a-4b2d-9c5a-15ab383b0e4a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:21.994Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"01cd8862-2ace-4d1a-9939-06692e3e96fa"}
{"level":"info","ts":"2026-09-18T10:09:21.995Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"01cd8862-2ace-4d1a-9939-06692e3e96fa"}
{"level":"info","ts":"2026-09-18T10:09:21.995Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"01cd8862-2ace-4d1a-9939-06692e3e96fa"}
{"level":"info","ts":"2026-09-18T10:09:21.995Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"01cd8862-2ace-4d1a-9939-06692e3e96fa","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:21.995Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"01cd8862-2ace-4d1a-9939-06692e3e96fa","error":"head certificate secret ray-head-secret-counter-auth-hw6hh not found"}
{"level":"error","ts":"2026-09-18T10:09:21.995Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"01cd8862-2ace-4d1a-9939-06692e3e96fa","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:21.995Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"01cd8862-2ace-4d1a-9939-06692e3e96fa","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found"}
{"level":"info","ts":"2026-09-18T10:09:21.995Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"01cd8862-2ace-4d1a-9939-06692e3e96fa","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:21.995Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"01cd8862-2ace-4d1a-9939-06692e3e96fa","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:21.995Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"01cd8862-2ace-4d1a-9939-06692e3e96fa","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:21.995Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"01cd8862-2ace-4d1a-9939-06692e3e96fa","name":"counter-auth-hw6hh","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:09:22.000Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"80fc35d4-bcf5-43ed-8cc8-e60b737b8d95","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.000Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"ee422d2e-814a-4542-acf8-691e3a504bd0","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.005Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"db1d05f9-4654-446d-88bd-8ae7c2a30f25","cluster":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:22.005Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"0e591e07-7e9a-448f-9102-175763698276","namespacedName":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:22.005Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"0e591e07-7e9a-448f-9102-175763698276","mode":"IntegratedOAuth","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.005Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"0e591e07-7e9a-448f-9102-175763698276","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.005Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"0e591e07-7e9a-448f-9102-175763698276","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.067Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"cd1c14b1-df10-4da1-9f51-1688d6f061a9","rayCluster":"counter-auth-hw6hh","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T10:09:22.067Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"cd1c14b1-df10-4da1-9f51-1688d6f061a9","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.067Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"01cd8862-2ace-4d1a-9939-06692e3e96fa"}
{"level":"error","ts":"2026-09-18T10:09:22.067Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"01cd8862-2ace-4d1a-9939-06692e3e96fa","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:22.067Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"11e66421-5cee-4db2-bfa5-b7f27232b913"}
{"level":"info","ts":"2026-09-18T10:09:22.067Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"11e66421-5cee-4db2-bfa5-b7f27232b913"}
{"level":"info","ts":"2026-09-18T10:09:22.067Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"11e66421-5cee-4db2-bfa5-b7f27232b913"}
{"level":"info","ts":"2026-09-18T10:09:22.067Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"11e66421-5cee-4db2-bfa5-b7f27232b913","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:22.067Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"11e66421-5cee-4db2-bfa5-b7f27232b913","error":"head certificate secret ray-head-secret-counter-auth-hw6hh not found"}
{"level":"info","ts":"2026-09-18T10:09:22.067Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"06aaf6bc-a848-4f5d-a500-b19bd909fb40"}
{"level":"error","ts":"2026-09-18T10:09:22.067Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"11e66421-5cee-4db2-bfa5-b7f27232b913","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:22.067Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"11e66421-5cee-4db2-bfa5-b7f27232b913","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found"}
{"level":"info","ts":"2026-09-18T10:09:22.068Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"2422779e-76dd-49d1-8cd1-cd3a5c656b24","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:22.068Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"2422779e-76dd-49d1-8cd1-cd3a5c656b24","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.068Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"2422779e-76dd-49d1-8cd1-cd3a5c656b24","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:22.068Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"2422779e-76dd-49d1-8cd1-cd3a5c656b24"}
{"level":"info","ts":"2026-09-18T10:09:22.068Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"2422779e-76dd-49d1-8cd1-cd3a5c656b24","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:22.068Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"2422779e-76dd-49d1-8cd1-cd3a5c656b24","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:22.068Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"2422779e-76dd-49d1-8cd1-cd3a5c656b24","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:09:22.069Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"11e66421-5cee-4db2-bfa5-b7f27232b913","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.069Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"11e66421-5cee-4db2-bfa5-b7f27232b913","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.069Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"11e66421-5cee-4db2-bfa5-b7f27232b913","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.069Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"11e66421-5cee-4db2-bfa5-b7f27232b913","name":"counter-auth-hw6hh","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:22Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:22Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:22Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:22Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:09:22.168Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.168Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:09:22.169Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"cd1c14b1-df10-4da1-9f51-1688d6f061a9","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.170Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e4ed2f00-31cc-4561-9aa0-60c7c3e1d394","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:22.170Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e4ed2f00-31cc-4561-9aa0-60c7c3e1d394","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.170Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e4ed2f00-31cc-4561-9aa0-60c7c3e1d394","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:22.170Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e4ed2f00-31cc-4561-9aa0-60c7c3e1d394"}
{"level":"info","ts":"2026-09-18T10:09:22.170Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e4ed2f00-31cc-4561-9aa0-60c7c3e1d394","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:22.170Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e4ed2f00-31cc-4561-9aa0-60c7c3e1d394","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:22.266Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"11e66421-5cee-4db2-bfa5-b7f27232b913","name":"counter-auth-hw6hh","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-hw6hh\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"counter-auth-hw6hh"}}
{"level":"info","ts":"2026-09-18T10:09:22.266Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"11e66421-5cee-4db2-bfa5-b7f27232b913"}
{"level":"error","ts":"2026-09-18T10:09:22.266Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"11e66421-5cee-4db2-bfa5-b7f27232b913","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:22.266Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"18a1c355-7232-4ca4-b051-f8b0433ad44f"}
{"level":"info","ts":"2026-09-18T10:09:22.266Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"18a1c355-7232-4ca4-b051-f8b0433ad44f"}
{"level":"info","ts":"2026-09-18T10:09:22.266Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"18a1c355-7232-4ca4-b051-f8b0433ad44f"}
{"level":"info","ts":"2026-09-18T10:09:22.266Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"18a1c355-7232-4ca4-b051-f8b0433ad44f","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:22.266Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"18a1c355-7232-4ca4-b051-f8b0433ad44f","error":"head certificate secret ray-head-secret-counter-auth-hw6hh not found"}
{"level":"error","ts":"2026-09-18T10:09:22.266Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"18a1c355-7232-4ca4-b051-f8b0433ad44f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:22.266Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"18a1c355-7232-4ca4-b051-f8b0433ad44f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found"}
{"level":"info","ts":"2026-09-18T10:09:22.266Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"ee422d2e-814a-4542-acf8-691e3a504bd0","name":"counter-auth-hw6hh-head"}
{"level":"info","ts":"2026-09-18T10:09:22.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"18a1c355-7232-4ca4-b051-f8b0433ad44f","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"18a1c355-7232-4ca4-b051-f8b0433ad44f","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.267Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"18a1c355-7232-4ca4-b051-f8b0433ad44f","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.267Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"18a1c355-7232-4ca4-b051-f8b0433ad44f"}
{"level":"error","ts":"2026-09-18T10:09:22.267Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"18a1c355-7232-4ca4-b051-f8b0433ad44f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:22.267Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"cd1c14b1-df10-4da1-9f51-1688d6f061a9","rayCluster":"counter-auth-hw6hh","podIPs":[]}
{"level":"error","ts":"2026-09-18T10:09:22.268Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"0e591e07-7e9a-448f-9102-175763698276","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-hw6hh\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:22.268Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"0e591e07-7e9a-448f-9102-175763698276"}
{"level":"error","ts":"2026-09-18T10:09:22.268Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"0e591e07-7e9a-448f-9102-175763698276","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-hw6hh\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:22.268Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"31ae4730-53db-496d-a864-76507fa27f55","namespacedName":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:22.268Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"31ae4730-53db-496d-a864-76507fa27f55"}
{"level":"info","ts":"2026-09-18T10:09:22.268Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"31ae4730-53db-496d-a864-76507fa27f55","namespace":"test-ns-ltx56","authClusters":2}
{"level":"info","ts":"2026-09-18T10:09:22.268Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"31ae4730-53db-496d-a864-76507fa27f55","namespace":"test-ns-ltx56","authClusters":2}
{"level":"info","ts":"2026-09-18T10:09:22.366Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"91a586e4-aa7f-40b4-8f26-7ff2ae9cc5b9","namespacedName":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:22.366Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"3cb4a324-7d01-4d24-b97a-c0bb4aa8604b"}
{"level":"info","ts":"2026-09-18T10:09:22.366Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"91a586e4-aa7f-40b4-8f26-7ff2ae9cc5b9","mode":"IntegratedOAuth","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.366Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"91a586e4-aa7f-40b4-8f26-7ff2ae9cc5b9","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.366Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"3cb4a324-7d01-4d24-b97a-c0bb4aa8604b"}
{"level":"info","ts":"2026-09-18T10:09:22.366Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"91a586e4-aa7f-40b4-8f26-7ff2ae9cc5b9","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.366Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"3cb4a324-7d01-4d24-b97a-c0bb4aa8604b"}
{"level":"info","ts":"2026-09-18T10:09:22.366Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"3cb4a324-7d01-4d24-b97a-c0bb4aa8604b","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:22.366Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"3cb4a324-7d01-4d24-b97a-c0bb4aa8604b","error":"head certificate secret ray-head-secret-counter-auth-hw6hh not found"}
{"level":"error","ts":"2026-09-18T10:09:22.366Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"3cb4a324-7d01-4d24-b97a-c0bb4aa8604b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:22.366Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"3cb4a324-7d01-4d24-b97a-c0bb4aa8604b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found"}
{"level":"info","ts":"2026-09-18T10:09:22.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"3cb4a324-7d01-4d24-b97a-c0bb4aa8604b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"3cb4a324-7d01-4d24-b97a-c0bb4aa8604b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"3cb4a324-7d01-4d24-b97a-c0bb4aa8604b","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.366Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"3cb4a324-7d01-4d24-b97a-c0bb4aa8604b"}
{"level":"error","ts":"2026-09-18T10:09:22.366Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"3cb4a324-7d01-4d24-b97a-c0bb4aa8604b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:22.368Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"cd1c14b1-df10-4da1-9f51-1688d6f061a9","rayCluster":"counter-auth-hw6hh","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:09:22.368Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"cd1c14b1-df10-4da1-9f51-1688d6f061a9"}
{"level":"info","ts":"2026-09-18T10:09:22.368Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"c18b8b86-7556-48ef-b618-f1cd1c168d0b","namespace":"test-ns-ltx56","clusterName":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:22.374Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.374Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:09:22.375Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"ee422d2e-814a-4542-acf8-691e3a504bd0","name":"counter-auth-hw6hh-workers"}
{"level":"info","ts":"2026-09-18T10:09:22.375Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"ee422d2e-814a-4542-acf8-691e3a504bd0","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.375Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"6295bbac-fc84-49af-ad1f-40c4b8478f95"}
{"level":"info","ts":"2026-09-18T10:09:22.375Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"35bfd841-2c42-4839-b4c1-ecdf2f615e54","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.466Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"75782a4e-71bf-410f-91c7-3826ee385def"}
{"level":"info","ts":"2026-09-18T10:09:22.466Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"75782a4e-71bf-410f-91c7-3826ee385def"}
{"level":"info","ts":"2026-09-18T10:09:22.466Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"75782a4e-71bf-410f-91c7-3826ee385def"}
{"level":"info","ts":"2026-09-18T10:09:22.466Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"75782a4e-71bf-410f-91c7-3826ee385def","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:22.466Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"75782a4e-71bf-410f-91c7-3826ee385def","error":"head certificate secret ray-head-secret-counter-auth-hw6hh not found"}
{"level":"error","ts":"2026-09-18T10:09:22.466Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"75782a4e-71bf-410f-91c7-3826ee385def","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:22.466Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"75782a4e-71bf-410f-91c7-3826ee385def","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found"}
{"level":"info","ts":"2026-09-18T10:09:22.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"75782a4e-71bf-410f-91c7-3826ee385def","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"75782a4e-71bf-410f-91c7-3826ee385def","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"75782a4e-71bf-410f-91c7-3826ee385def","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.466Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"75782a4e-71bf-410f-91c7-3826ee385def"}
{"level":"error","ts":"2026-09-18T10:09:22.466Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"75782a4e-71bf-410f-91c7-3826ee385def","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:22.467Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"985e2cec-33ed-44df-94f1-cad6988ad887","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:22.467Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"985e2cec-33ed-44df-94f1-cad6988ad887","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.467Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"985e2cec-33ed-44df-94f1-cad6988ad887","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:22.467Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"985e2cec-33ed-44df-94f1-cad6988ad887"}
{"level":"info","ts":"2026-09-18T10:09:22.467Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"985e2cec-33ed-44df-94f1-cad6988ad887","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:22.467Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"985e2cec-33ed-44df-94f1-cad6988ad887","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:22.474Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"91a586e4-aa7f-40b4-8f26-7ff2ae9cc5b9","name":"counter-auth-hw6hh-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:09:22.474Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"35bfd841-2c42-4839-b4c1-ecdf2f615e54","name":"counter-auth-hw6hh-head"}
{"level":"info","ts":"2026-09-18T10:09:22.485Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"c18b8b86-7556-48ef-b618-f1cd1c168d0b","namespace":"test-ns-ltx56","clusterName":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:09:22.485Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"0b51d5c6-2dfa-49fe-880c-8d68e22f157d","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.485Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"0b51d5c6-2dfa-49fe-880c-8d68e22f157d"}
{"level":"info","ts":"2026-09-18T10:09:22.491Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"91a586e4-aa7f-40b4-8f26-7ff2ae9cc5b9","name":"test-ns-ltx56-counter-auth-hw6hh","operation":"created"}
{"level":"info","ts":"2026-09-18T10:09:22.499Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"35bfd841-2c42-4839-b4c1-ecdf2f615e54","name":"counter-auth-hw6hh-workers"}
{"level":"info","ts":"2026-09-18T10:09:22.499Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"35bfd841-2c42-4839-b4c1-ecdf2f615e54","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.499Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"91d3946f-176a-4125-afa7-c8404de23365","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.500Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"91a586e4-aa7f-40b4-8f26-7ff2ae9cc5b9","name":"kube-rbac-proxy-config-counter-auth-hw6hh","operation":"created"}
{"level":"info","ts":"2026-09-18T10:09:22.522Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"91d3946f-176a-4125-afa7-c8404de23365","name":"counter-auth-hw6hh-head"}
{"level":"info","ts":"2026-09-18T10:09:22.567Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"91a586e4-aa7f-40b4-8f26-7ff2ae9cc5b9","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.567Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"d0a02a46-2601-40bd-9dfe-37fb15ddd2ae","namespacedName":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:22.567Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"d0a02a46-2601-40bd-9dfe-37fb15ddd2ae","mode":"IntegratedOAuth","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.567Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"d0a02a46-2601-40bd-9dfe-37fb15ddd2ae","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.568Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"34903d64-fe0c-4410-99b7-7bed774917fc","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.568Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"34903d64-fe0c-4410-99b7-7bed774917fc"}
{"level":"info","ts":"2026-09-18T10:09:22.568Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b4afa72f-e4c6-4d01-81fa-94eb67349b5e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:22.568Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b4afa72f-e4c6-4d01-81fa-94eb67349b5e","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.568Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b4afa72f-e4c6-4d01-81fa-94eb67349b5e","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:22.568Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b4afa72f-e4c6-4d01-81fa-94eb67349b5e"}
{"level":"info","ts":"2026-09-18T10:09:22.568Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b4afa72f-e4c6-4d01-81fa-94eb67349b5e","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:22.568Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b4afa72f-e4c6-4d01-81fa-94eb67349b5e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:22.568Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b4afa72f-e4c6-4d01-81fa-94eb67349b5e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:09:22.575Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"91d3946f-176a-4125-afa7-c8404de23365","name":"counter-auth-hw6hh-workers"}
{"level":"info","ts":"2026-09-18T10:09:22.575Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"91d3946f-176a-4125-afa7-c8404de23365","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.575Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"d9d60a5e-a467-484f-b021-004317032b29","cluster":"counter-auth-hw6hh"}
{"level":"error","ts":"2026-09-18T10:09:22.588Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"d0a02a46-2601-40bd-9dfe-37fb15ddd2ae","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-hw6hh\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:22.588Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"d0a02a46-2601-40bd-9dfe-37fb15ddd2ae"}
{"level":"error","ts":"2026-09-18T10:09:22.588Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"d0a02a46-2601-40bd-9dfe-37fb15ddd2ae","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-hw6hh\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:22.594Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"4ea4c07f-bae6-4cca-b11c-fd3aad93703a","namespacedName":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:22.594Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"4ea4c07f-bae6-4cca-b11c-fd3aad93703a","mode":"IntegratedOAuth","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.594Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"4ea4c07f-bae6-4cca-b11c-fd3aad93703a","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.603Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"d9d60a5e-a467-484f-b021-004317032b29","name":"counter-auth-hw6hh-head"}
{"level":"info","ts":"2026-09-18T10:09:22.606Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"1787978c-669e-4871-82b4-3abea7021cfc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:22.606Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"1787978c-669e-4871-82b4-3abea7021cfc","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.606Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"1787978c-669e-4871-82b4-3abea7021cfc","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:22.606Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"1787978c-669e-4871-82b4-3abea7021cfc"}
{"level":"info","ts":"2026-09-18T10:09:22.606Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"1787978c-669e-4871-82b4-3abea7021cfc","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:22.606Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"1787978c-669e-4871-82b4-3abea7021cfc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:22.623Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"d9d60a5e-a467-484f-b021-004317032b29","name":"counter-auth-hw6hh-workers"}
{"level":"info","ts":"2026-09-18T10:09:22.623Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"d9d60a5e-a467-484f-b021-004317032b29","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.627Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"8e9db6da-ebab-42a4-9f0c-208117605ce3"}
{"level":"info","ts":"2026-09-18T10:09:22.627Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"8e9db6da-ebab-42a4-9f0c-208117605ce3"}
{"level":"info","ts":"2026-09-18T10:09:22.627Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"8e9db6da-ebab-42a4-9f0c-208117605ce3"}
{"level":"info","ts":"2026-09-18T10:09:22.627Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"8e9db6da-ebab-42a4-9f0c-208117605ce3","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:22.627Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"8e9db6da-ebab-42a4-9f0c-208117605ce3","error":"head certificate secret ray-head-secret-counter-auth-hw6hh not found"}
{"level":"error","ts":"2026-09-18T10:09:22.627Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"8e9db6da-ebab-42a4-9f0c-208117605ce3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:22.627Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"8e9db6da-ebab-42a4-9f0c-208117605ce3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found"}
{"level":"info","ts":"2026-09-18T10:09:22.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"8e9db6da-ebab-42a4-9f0c-208117605ce3","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"8e9db6da-ebab-42a4-9f0c-208117605ce3","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"8e9db6da-ebab-42a4-9f0c-208117605ce3","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.627Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"8e9db6da-ebab-42a4-9f0c-208117605ce3"}
{"level":"error","ts":"2026-09-18T10:09:22.627Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"8e9db6da-ebab-42a4-9f0c-208117605ce3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:22.666Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"4ea4c07f-bae6-4cca-b11c-fd3aad93703a","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:22.948Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"1e5432ae-5553-4e31-94c1-3490f6238e35"}
{"level":"info","ts":"2026-09-18T10:09:22.948Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"1e5432ae-5553-4e31-94c1-3490f6238e35"}
{"level":"info","ts":"2026-09-18T10:09:22.948Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"1e5432ae-5553-4e31-94c1-3490f6238e35"}
{"level":"info","ts":"2026-09-18T10:09:22.948Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"1e5432ae-5553-4e31-94c1-3490f6238e35","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:22.948Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"1e5432ae-5553-4e31-94c1-3490f6238e35","error":"head certificate secret ray-head-secret-counter-auth-hw6hh not found"}
{"level":"error","ts":"2026-09-18T10:09:22.948Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"1e5432ae-5553-4e31-94c1-3490f6238e35","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:22.948Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"1e5432ae-5553-4e31-94c1-3490f6238e35","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found"}
{"level":"info","ts":"2026-09-18T10:09:22.948Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"1e5432ae-5553-4e31-94c1-3490f6238e35","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.948Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"1e5432ae-5553-4e31-94c1-3490f6238e35","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.948Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"1e5432ae-5553-4e31-94c1-3490f6238e35","filter labels":["test-ns-ltx56",{"ray.io/cluster":"counter-auth-hw6hh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:09:22.949Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"1e5432ae-5553-4e31-94c1-3490f6238e35"}
{"level":"error","ts":"2026-09-18T10:09:22.949Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"1e5432ae-5553-4e31-94c1-3490f6238e35","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-auth-hw6hh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:23.283Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-ltx56"},"reconcileID":"77d746f7-4d26-48e3-a7d5-253a1bb03117"}
{"level":"info","ts":"2026-09-18T10:09:23.589Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66"}
{"level":"info","ts":"2026-09-18T10:09:23.589Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66"}
{"level":"info","ts":"2026-09-18T10:09:23.589Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66"}
{"level":"info","ts":"2026-09-18T10:09:23.590Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:23.590Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66"}
{"level":"info","ts":"2026-09-18T10:09:23.590Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:09:23.590Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66"}
{"level":"info","ts":"2026-09-18T10:09:23.590Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","mode":"IntegratedOAuth","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:23.590Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","cluster":"counter-auth-hw6hh","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:09:23.590Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","cluster":"counter-auth-hw6hh","authType":"IntegratedOAuth","serviceAccount":"counter-auth-hw6hh-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T10:09:23.590Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-auth-hw6hh","ray.io/group":"headgroup","ray.io/identifier":"counter-auth-hw6hh-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:09:23.590Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T10:09:23.590Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:09:23.590Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:09:23.590Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","enabled":true}
{"level":"info","ts":"2026-09-18T10:09:23.629Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","name":"counter-auth-hw6hh-head-ssjn9"}
{"level":"info","ts":"2026-09-18T10:09:23.629Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:23.629Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:23.629Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:09:23.629Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:09:23.629Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:23.629Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:09:23.629Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66"}
{"level":"info","ts":"2026-09-18T10:09:23.629Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","nodeType":"worker","rayStartParams":{"address":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:09:23.629Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","rayStartCmd":"ray start  --address=counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:09:23.629Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:09:23.629Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","enabled":true}
{"level":"info","ts":"2026-09-18T10:09:23.667Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","name":"counter-auth-hw6hh-small-group-worker-9mpg4"}
{"level":"info","ts":"2026-09-18T10:09:23.667Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:23.667Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:23.668Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ff6795d8-f3fa-4baa-b327-f7442248ee66","name":"counter-auth-hw6hh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:09:23.707Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"daf7cc78-4ed4-4cf7-b288-dbf49d673d8f","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:23.707Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6c408c72-c35c-4bd2-9d3e-02b90cf11701","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:23.707Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6c408c72-c35c-4bd2-9d3e-02b90cf11701","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:23.707Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6c408c72-c35c-4bd2-9d3e-02b90cf11701","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:23.707Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"c0251f59-8a04-4aa0-a779-de87ddc91bbe","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:23.707Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6c408c72-c35c-4bd2-9d3e-02b90cf11701"}
{"level":"info","ts":"2026-09-18T10:09:23.707Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6c408c72-c35c-4bd2-9d3e-02b90cf11701","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:23.707Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6c408c72-c35c-4bd2-9d3e-02b90cf11701","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:23.707Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6c408c72-c35c-4bd2-9d3e-02b90cf11701","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:09:23.707Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"daf7cc78-4ed4-4cf7-b288-dbf49d673d8f"}
{"level":"info","ts":"2026-09-18T10:09:23.711Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e0a1cd6-e471-4a4a-9b69-4d0f4b8be0c9"}
{"level":"info","ts":"2026-09-18T10:09:23.711Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e0a1cd6-e471-4a4a-9b69-4d0f4b8be0c9"}
{"level":"info","ts":"2026-09-18T10:09:23.711Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e0a1cd6-e471-4a4a-9b69-4d0f4b8be0c9"}
{"level":"info","ts":"2026-09-18T10:09:23.711Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e0a1cd6-e471-4a4a-9b69-4d0f4b8be0c9","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:23.711Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e0a1cd6-e471-4a4a-9b69-4d0f4b8be0c9","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:23.711Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e0a1cd6-e471-4a4a-9b69-4d0f4b8be0c9","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:23.711Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e0a1cd6-e471-4a4a-9b69-4d0f4b8be0c9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:23.711Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e0a1cd6-e471-4a4a-9b69-4d0f4b8be0c9","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:23.711Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e0a1cd6-e471-4a4a-9b69-4d0f4b8be0c9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:23.711Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e0a1cd6-e471-4a4a-9b69-4d0f4b8be0c9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:23.711Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e0a1cd6-e471-4a4a-9b69-4d0f4b8be0c9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:23.711Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e0a1cd6-e471-4a4a-9b69-4d0f4b8be0c9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:23.711Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e0a1cd6-e471-4a4a-9b69-4d0f4b8be0c9","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:23.711Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e0a1cd6-e471-4a4a-9b69-4d0f4b8be0c9","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:23.712Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e0a1cd6-e471-4a4a-9b69-4d0f4b8be0c9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:09:23.712Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e0a1cd6-e471-4a4a-9b69-4d0f4b8be0c9","seconds":300}
{"level":"info","ts":"2026-09-18T10:09:23.718Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"08038d54-fe1e-4e9d-ba11-50cd4b2fd20c"}
{"level":"info","ts":"2026-09-18T10:09:23.718Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"08038d54-fe1e-4e9d-ba11-50cd4b2fd20c"}
{"level":"info","ts":"2026-09-18T10:09:23.718Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"08038d54-fe1e-4e9d-ba11-50cd4b2fd20c"}
{"level":"info","ts":"2026-09-18T10:09:23.718Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"08038d54-fe1e-4e9d-ba11-50cd4b2fd20c","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:23.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"08038d54-fe1e-4e9d-ba11-50cd4b2fd20c","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:23.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"08038d54-fe1e-4e9d-ba11-50cd4b2fd20c","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:23.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"08038d54-fe1e-4e9d-ba11-50cd4b2fd20c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:23.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"08038d54-fe1e-4e9d-ba11-50cd4b2fd20c","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:23.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"08038d54-fe1e-4e9d-ba11-50cd4b2fd20c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:23.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"08038d54-fe1e-4e9d-ba11-50cd4b2fd20c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:23.718Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"08038d54-fe1e-4e9d-ba11-50cd4b2fd20c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:23.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"08038d54-fe1e-4e9d-ba11-50cd4b2fd20c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:23.718Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"08038d54-fe1e-4e9d-ba11-50cd4b2fd20c","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:23.718Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"08038d54-fe1e-4e9d-ba11-50cd4b2fd20c","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:23.719Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"08038d54-fe1e-4e9d-ba11-50cd4b2fd20c","name":"counter-auth-hw6hh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:09:23.766Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"c0251f59-8a04-4aa0-a779-de87ddc91bbe","name":"counter-auth-hw6hh-head"}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"987be2b4-368b-40e8-8259-fb0a0b45fd82"}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"987be2b4-368b-40e8-8259-fb0a0b45fd82"}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"987be2b4-368b-40e8-8259-fb0a0b45fd82"}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"987be2b4-368b-40e8-8259-fb0a0b45fd82","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"4d2e6a1d-a54c-4a6b-beed-79cc0a19a469","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"4d2e6a1d-a54c-4a6b-beed-79cc0a19a469","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"4d2e6a1d-a54c-4a6b-beed-79cc0a19a469","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"987be2b4-368b-40e8-8259-fb0a0b45fd82","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"987be2b4-368b-40e8-8259-fb0a0b45fd82","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"987be2b4-368b-40e8-8259-fb0a0b45fd82","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"987be2b4-368b-40e8-8259-fb0a0b45fd82","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"987be2b4-368b-40e8-8259-fb0a0b45fd82","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"987be2b4-368b-40e8-8259-fb0a0b45fd82","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"987be2b4-368b-40e8-8259-fb0a0b45fd82","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"987be2b4-368b-40e8-8259-fb0a0b45fd82","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"987be2b4-368b-40e8-8259-fb0a0b45fd82","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"987be2b4-368b-40e8-8259-fb0a0b45fd82","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"987be2b4-368b-40e8-8259-fb0a0b45fd82","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"987be2b4-368b-40e8-8259-fb0a0b45fd82","seconds":300}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"4d2e6a1d-a54c-4a6b-beed-79cc0a19a469"}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"4d2e6a1d-a54c-4a6b-beed-79cc0a19a469","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"4d2e6a1d-a54c-4a6b-beed-79cc0a19a469","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"4d2e6a1d-a54c-4a6b-beed-79cc0a19a469","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:09:23.767Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"b690ad37-c59c-42fd-a903-ce85b09cb0b2","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:23.768Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"b690ad37-c59c-42fd-a903-ce85b09cb0b2"}
{"level":"info","ts":"2026-09-18T10:09:23.768Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"879f3dc4-6cf4-4ca6-9188-622c9d2a22ef"}
{"level":"info","ts":"2026-09-18T10:09:23.768Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"879f3dc4-6cf4-4ca6-9188-622c9d2a22ef"}
{"level":"info","ts":"2026-09-18T10:09:23.768Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"879f3dc4-6cf4-4ca6-9188-622c9d2a22ef"}
{"level":"info","ts":"2026-09-18T10:09:23.768Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"879f3dc4-6cf4-4ca6-9188-622c9d2a22ef","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:23.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"879f3dc4-6cf4-4ca6-9188-622c9d2a22ef","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:23.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"879f3dc4-6cf4-4ca6-9188-622c9d2a22ef","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:23.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"879f3dc4-6cf4-4ca6-9188-622c9d2a22ef","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:23.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"879f3dc4-6cf4-4ca6-9188-622c9d2a22ef","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:23.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"879f3dc4-6cf4-4ca6-9188-622c9d2a22ef","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:23.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"879f3dc4-6cf4-4ca6-9188-622c9d2a22ef","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:23.768Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"879f3dc4-6cf4-4ca6-9188-622c9d2a22ef","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:23.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"879f3dc4-6cf4-4ca6-9188-622c9d2a22ef","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:23.768Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"879f3dc4-6cf4-4ca6-9188-622c9d2a22ef","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:23.768Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"879f3dc4-6cf4-4ca6-9188-622c9d2a22ef","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:23.768Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"879f3dc4-6cf4-4ca6-9188-622c9d2a22ef","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:09:23.769Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"879f3dc4-6cf4-4ca6-9188-622c9d2a22ef","seconds":300}
{"level":"info","ts":"2026-09-18T10:09:23.780Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"633818a3-3c24-40f6-b85d-e752569a1557"}
{"level":"info","ts":"2026-09-18T10:09:23.780Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"633818a3-3c24-40f6-b85d-e752569a1557"}
{"level":"info","ts":"2026-09-18T10:09:23.780Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"633818a3-3c24-40f6-b85d-e752569a1557"}
{"level":"info","ts":"2026-09-18T10:09:23.780Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"633818a3-3c24-40f6-b85d-e752569a1557","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:23.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"633818a3-3c24-40f6-b85d-e752569a1557","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:23.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"633818a3-3c24-40f6-b85d-e752569a1557","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:23.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"633818a3-3c24-40f6-b85d-e752569a1557","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:23.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"633818a3-3c24-40f6-b85d-e752569a1557","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:23.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"633818a3-3c24-40f6-b85d-e752569a1557","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:23.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"633818a3-3c24-40f6-b85d-e752569a1557","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:23.780Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"633818a3-3c24-40f6-b85d-e752569a1557","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:23.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"633818a3-3c24-40f6-b85d-e752569a1557","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:23.780Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"633818a3-3c24-40f6-b85d-e752569a1557","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:23.780Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"633818a3-3c24-40f6-b85d-e752569a1557","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:23.780Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"633818a3-3c24-40f6-b85d-e752569a1557","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:09:23.780Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"633818a3-3c24-40f6-b85d-e752569a1557","seconds":300}
{"level":"info","ts":"2026-09-18T10:09:23.784Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"c0251f59-8a04-4aa0-a779-de87ddc91bbe","name":"counter-auth-hw6hh-workers"}
{"level":"info","ts":"2026-09-18T10:09:23.784Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"c0251f59-8a04-4aa0-a779-de87ddc91bbe","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:23.784Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"4c3de2f2-4f31-4b8c-ac42-948578992c0b","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:23.791Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"19ed14f1-5bff-4b63-9fe7-c6d96cce2ba6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:23.791Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"19ed14f1-5bff-4b63-9fe7-c6d96cce2ba6","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:23.792Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"19ed14f1-5bff-4b63-9fe7-c6d96cce2ba6","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:23.792Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"19ed14f1-5bff-4b63-9fe7-c6d96cce2ba6"}
{"level":"info","ts":"2026-09-18T10:09:23.792Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"19ed14f1-5bff-4b63-9fe7-c6d96cce2ba6","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:23.792Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"19ed14f1-5bff-4b63-9fe7-c6d96cce2ba6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:23.792Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"19ed14f1-5bff-4b63-9fe7-c6d96cce2ba6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:09:23.801Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"4c3de2f2-4f31-4b8c-ac42-948578992c0b","name":"counter-auth-hw6hh-head"}
{"level":"info","ts":"2026-09-18T10:09:23.804Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"19ed14f1-5bff-4b63-9fe7-c6d96cce2ba6","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter-auth\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T10:09:23.804Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"19ed14f1-5bff-4b63-9fe7-c6d96cce2ba6"}
{"level":"error","ts":"2026-09-18T10:09:23.804Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"19ed14f1-5bff-4b63-9fe7-c6d96cce2ba6","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:09:23.804Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"90e41cb8-0bde-4f97-9d4f-cf29f85f957f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:23.804Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"90e41cb8-0bde-4f97-9d4f-cf29f85f957f","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:23.804Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"90e41cb8-0bde-4f97-9d4f-cf29f85f957f","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:23.819Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"4c3de2f2-4f31-4b8c-ac42-948578992c0b","name":"counter-auth-hw6hh-workers"}
{"level":"info","ts":"2026-09-18T10:09:23.819Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"4c3de2f2-4f31-4b8c-ac42-948578992c0b","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:23.865Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"90e41cb8-0bde-4f97-9d4f-cf29f85f957f"}
{"level":"info","ts":"2026-09-18T10:09:23.865Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"90e41cb8-0bde-4f97-9d4f-cf29f85f957f","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:23.866Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"90e41cb8-0bde-4f97-9d4f-cf29f85f957f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:23.866Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"cf3ee6ed-2c56-42a6-b120-11eaa163bf08","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:23.866Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"cf3ee6ed-2c56-42a6-b120-11eaa163bf08","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:23.866Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"cf3ee6ed-2c56-42a6-b120-11eaa163bf08","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:23.866Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"cf3ee6ed-2c56-42a6-b120-11eaa163bf08"}
{"level":"info","ts":"2026-09-18T10:09:23.866Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"cf3ee6ed-2c56-42a6-b120-11eaa163bf08","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:23.866Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"cf3ee6ed-2c56-42a6-b120-11eaa163bf08","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:24.256Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5afc8719-a9df-4fe9-8fe9-ff2e8c389bb0"}
{"level":"info","ts":"2026-09-18T10:09:24.256Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5afc8719-a9df-4fe9-8fe9-ff2e8c389bb0"}
{"level":"info","ts":"2026-09-18T10:09:24.256Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5afc8719-a9df-4fe9-8fe9-ff2e8c389bb0"}
{"level":"info","ts":"2026-09-18T10:09:24.256Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5afc8719-a9df-4fe9-8fe9-ff2e8c389bb0","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:24.256Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5afc8719-a9df-4fe9-8fe9-ff2e8c389bb0","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:24.256Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5afc8719-a9df-4fe9-8fe9-ff2e8c389bb0","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:24.256Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5afc8719-a9df-4fe9-8fe9-ff2e8c389bb0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:24.256Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5afc8719-a9df-4fe9-8fe9-ff2e8c389bb0","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:24.256Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5afc8719-a9df-4fe9-8fe9-ff2e8c389bb0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:24.256Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5afc8719-a9df-4fe9-8fe9-ff2e8c389bb0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:24.256Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5afc8719-a9df-4fe9-8fe9-ff2e8c389bb0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:24.256Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5afc8719-a9df-4fe9-8fe9-ff2e8c389bb0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:24.256Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5afc8719-a9df-4fe9-8fe9-ff2e8c389bb0","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:24.256Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5afc8719-a9df-4fe9-8fe9-ff2e8c389bb0","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:24.256Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5afc8719-a9df-4fe9-8fe9-ff2e8c389bb0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:09:24.256Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5afc8719-a9df-4fe9-8fe9-ff2e8c389bb0","seconds":300}
{"level":"info","ts":"2026-09-18T10:09:24.300Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"a747abe4-6c15-42e3-b125-5c66bf08c1e7"}
{"level":"info","ts":"2026-09-18T10:09:24.300Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"a747abe4-6c15-42e3-b125-5c66bf08c1e7"}
{"level":"info","ts":"2026-09-18T10:09:24.300Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"a747abe4-6c15-42e3-b125-5c66bf08c1e7"}
{"level":"info","ts":"2026-09-18T10:09:24.300Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"a747abe4-6c15-42e3-b125-5c66bf08c1e7","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:24.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"a747abe4-6c15-42e3-b125-5c66bf08c1e7","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:24.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"a747abe4-6c15-42e3-b125-5c66bf08c1e7","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:24.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"a747abe4-6c15-42e3-b125-5c66bf08c1e7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:24.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"a747abe4-6c15-42e3-b125-5c66bf08c1e7","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:24.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"a747abe4-6c15-42e3-b125-5c66bf08c1e7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:24.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"a747abe4-6c15-42e3-b125-5c66bf08c1e7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:24.300Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"a747abe4-6c15-42e3-b125-5c66bf08c1e7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:24.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"a747abe4-6c15-42e3-b125-5c66bf08c1e7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:24.300Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"a747abe4-6c15-42e3-b125-5c66bf08c1e7","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:24.300Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"a747abe4-6c15-42e3-b125-5c66bf08c1e7","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:24.301Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"a747abe4-6c15-42e3-b125-5c66bf08c1e7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:09:24.301Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"a747abe4-6c15-42e3-b125-5c66bf08c1e7","seconds":300}
{"level":"info","ts":"2026-09-18T10:09:25.039Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"b6ac633d-24b4-4551-91dc-aeff3dac5cb2"}
{"level":"info","ts":"2026-09-18T10:09:25.039Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"b6ac633d-24b4-4551-91dc-aeff3dac5cb2"}
{"level":"info","ts":"2026-09-18T10:09:25.039Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"b6ac633d-24b4-4551-91dc-aeff3dac5cb2"}
{"level":"info","ts":"2026-09-18T10:09:25.039Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"b6ac633d-24b4-4551-91dc-aeff3dac5cb2","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:25.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"b6ac633d-24b4-4551-91dc-aeff3dac5cb2","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:25.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"b6ac633d-24b4-4551-91dc-aeff3dac5cb2","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:25.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"b6ac633d-24b4-4551-91dc-aeff3dac5cb2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:25.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"b6ac633d-24b4-4551-91dc-aeff3dac5cb2","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:25.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"b6ac633d-24b4-4551-91dc-aeff3dac5cb2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:25.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"b6ac633d-24b4-4551-91dc-aeff3dac5cb2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:25.039Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"b6ac633d-24b4-4551-91dc-aeff3dac5cb2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:25.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"b6ac633d-24b4-4551-91dc-aeff3dac5cb2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:25.040Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"b6ac633d-24b4-4551-91dc-aeff3dac5cb2","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:25.040Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"b6ac633d-24b4-4551-91dc-aeff3dac5cb2","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:25.040Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"b6ac633d-24b4-4551-91dc-aeff3dac5cb2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:09:25.040Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"b6ac633d-24b4-4551-91dc-aeff3dac5cb2","seconds":300}
{"level":"info","ts":"2026-09-18T10:09:25.229Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ac5c1008-b663-400d-8ec7-39084bb19e61"}
{"level":"info","ts":"2026-09-18T10:09:25.229Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ac5c1008-b663-400d-8ec7-39084bb19e61"}
{"level":"info","ts":"2026-09-18T10:09:25.229Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ac5c1008-b663-400d-8ec7-39084bb19e61"}
{"level":"info","ts":"2026-09-18T10:09:25.229Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ac5c1008-b663-400d-8ec7-39084bb19e61","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:25.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ac5c1008-b663-400d-8ec7-39084bb19e61","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:25.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ac5c1008-b663-400d-8ec7-39084bb19e61","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:25.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ac5c1008-b663-400d-8ec7-39084bb19e61","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:25.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ac5c1008-b663-400d-8ec7-39084bb19e61","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:25.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ac5c1008-b663-400d-8ec7-39084bb19e61","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:25.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ac5c1008-b663-400d-8ec7-39084bb19e61","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:25.230Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ac5c1008-b663-400d-8ec7-39084bb19e61","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:25.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ac5c1008-b663-400d-8ec7-39084bb19e61","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:25.230Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ac5c1008-b663-400d-8ec7-39084bb19e61","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:25.230Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ac5c1008-b663-400d-8ec7-39084bb19e61","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:25.230Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"ac5c1008-b663-400d-8ec7-39084bb19e61","name":"counter-auth-hw6hh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:09:25.245Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f435626f-31f4-481e-87fb-1094eba22449","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:25.245Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f435626f-31f4-481e-87fb-1094eba22449","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:25.245Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f435626f-31f4-481e-87fb-1094eba22449","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:25.245Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"16e0142c-11a7-4658-a4d6-1878bda2da9a","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:25.245Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"16e0142c-11a7-4658-a4d6-1878bda2da9a","pod":"counter-auth-hw6hh-head-ssjn9","ip":"10.133.0.48"}
{"level":"info","ts":"2026-09-18T10:09:25.245Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"16e0142c-11a7-4658-a4d6-1878bda2da9a","pod":"counter-auth-hw6hh-small-group-worker-9mpg4","ip":"10.132.0.55"}
{"level":"info","ts":"2026-09-18T10:09:25.245Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f435626f-31f4-481e-87fb-1094eba22449"}
{"level":"info","ts":"2026-09-18T10:09:25.245Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f435626f-31f4-481e-87fb-1094eba22449","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:25.245Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f435626f-31f4-481e-87fb-1094eba22449","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:25.245Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f435626f-31f4-481e-87fb-1094eba22449","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:09:25.245Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"ce7e9268-5ef0-48b2-a036-77fc9158200b","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:25.259Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"ce7e9268-5ef0-48b2-a036-77fc9158200b","name":"counter-auth-hw6hh-head"}
{"level":"info","ts":"2026-09-18T10:09:25.260Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"16e0142c-11a7-4658-a4d6-1878bda2da9a","certificate":"ray-head-cert-counter-auth-hw6hh","dnsNames":["counter-auth-hw6hh-head-svc","localhost","counter-auth-hw6hh-head-svc.test-ns-ltx56.svc","counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.48","10.132.0.55","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:09:25.265Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"34a4db69-5b86-4e9a-b041-7bbd69ff99ac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:25.265Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"34a4db69-5b86-4e9a-b041-7bbd69ff99ac","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:25.265Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"34a4db69-5b86-4e9a-b041-7bbd69ff99ac","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:25.265Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"34a4db69-5b86-4e9a-b041-7bbd69ff99ac"}
{"level":"info","ts":"2026-09-18T10:09:25.265Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"34a4db69-5b86-4e9a-b041-7bbd69ff99ac","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:25.265Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"34a4db69-5b86-4e9a-b041-7bbd69ff99ac","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:25.279Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"16e0142c-11a7-4658-a4d6-1878bda2da9a","certificate":"ray-worker-cert-counter-auth-hw6hh","dnsNames":["counter-auth-hw6hh-worker-svc","localhost","counter-auth-hw6hh-small-group","counter-auth-hw6hh-worker-svc.test-ns-ltx56.svc","counter-auth-hw6hh-worker-svc.test-ns-ltx56.svc.cluster.local","*.counter-auth-hw6hh-worker-svc.test-ns-ltx56.svc","*.counter-auth-hw6hh-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.48","10.132.0.55","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:09:25.279Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"16e0142c-11a7-4658-a4d6-1878bda2da9a","rayCluster":"counter-auth-hw6hh","podIPs":["10.133.0.48","10.132.0.55"]}
{"level":"info","ts":"2026-09-18T10:09:25.282Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"ce7e9268-5ef0-48b2-a036-77fc9158200b","name":"counter-auth-hw6hh-workers"}
{"level":"info","ts":"2026-09-18T10:09:25.282Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"ce7e9268-5ef0-48b2-a036-77fc9158200b","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:25.712Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"21824519-9a57-40e6-8a23-0802518e0352"}
{"level":"info","ts":"2026-09-18T10:09:25.712Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"21824519-9a57-40e6-8a23-0802518e0352"}
{"level":"info","ts":"2026-09-18T10:09:25.712Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"21824519-9a57-40e6-8a23-0802518e0352"}
{"level":"info","ts":"2026-09-18T10:09:25.712Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"21824519-9a57-40e6-8a23-0802518e0352","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:25.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"21824519-9a57-40e6-8a23-0802518e0352","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:25.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"21824519-9a57-40e6-8a23-0802518e0352","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:25.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"21824519-9a57-40e6-8a23-0802518e0352","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:25.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"21824519-9a57-40e6-8a23-0802518e0352","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:25.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"21824519-9a57-40e6-8a23-0802518e0352","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:25.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"21824519-9a57-40e6-8a23-0802518e0352","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:25.712Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"21824519-9a57-40e6-8a23-0802518e0352","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:25.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"21824519-9a57-40e6-8a23-0802518e0352","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:25.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"21824519-9a57-40e6-8a23-0802518e0352","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:25.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"21824519-9a57-40e6-8a23-0802518e0352","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:25.712Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"21824519-9a57-40e6-8a23-0802518e0352","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:09:25.712Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"21824519-9a57-40e6-8a23-0802518e0352","seconds":300}
{"level":"info","ts":"2026-09-18T10:09:26.867Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"38aa18cf-1b98-40fa-ac26-3980a507c00a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:26.867Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"38aa18cf-1b98-40fa-ac26-3980a507c00a","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:26.867Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"38aa18cf-1b98-40fa-ac26-3980a507c00a","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:26.867Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"38aa18cf-1b98-40fa-ac26-3980a507c00a"}
{"level":"info","ts":"2026-09-18T10:09:26.867Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"38aa18cf-1b98-40fa-ac26-3980a507c00a","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:26.867Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"38aa18cf-1b98-40fa-ac26-3980a507c00a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:29.868Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a2f41b9a-ff59-45a2-8637-833a4ba0aded","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:29.868Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a2f41b9a-ff59-45a2-8637-833a4ba0aded","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:29.868Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a2f41b9a-ff59-45a2-8637-833a4ba0aded","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:29.868Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a2f41b9a-ff59-45a2-8637-833a4ba0aded"}
{"level":"info","ts":"2026-09-18T10:09:29.868Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a2f41b9a-ff59-45a2-8637-833a4ba0aded","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:29.868Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a2f41b9a-ff59-45a2-8637-833a4ba0aded","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:30.255Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"38bc70f5-62b0-4d91-8182-152920717107"}
{"level":"info","ts":"2026-09-18T10:09:30.255Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"38bc70f5-62b0-4d91-8182-152920717107"}
{"level":"info","ts":"2026-09-18T10:09:30.255Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"38bc70f5-62b0-4d91-8182-152920717107"}
{"level":"info","ts":"2026-09-18T10:09:30.255Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"38bc70f5-62b0-4d91-8182-152920717107","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:30.255Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"38bc70f5-62b0-4d91-8182-152920717107","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:30.255Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"38bc70f5-62b0-4d91-8182-152920717107","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:30.255Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"38bc70f5-62b0-4d91-8182-152920717107","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:30.255Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"38bc70f5-62b0-4d91-8182-152920717107","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:30.255Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"38bc70f5-62b0-4d91-8182-152920717107","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:30.255Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"38bc70f5-62b0-4d91-8182-152920717107","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:30.255Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"38bc70f5-62b0-4d91-8182-152920717107","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:30.256Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"38bc70f5-62b0-4d91-8182-152920717107","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:30.256Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"38bc70f5-62b0-4d91-8182-152920717107","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:30.256Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"38bc70f5-62b0-4d91-8182-152920717107","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:30.256Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"38bc70f5-62b0-4d91-8182-152920717107","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:09:30.256Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"38bc70f5-62b0-4d91-8182-152920717107","seconds":300}
{"level":"info","ts":"2026-09-18T10:09:31.270Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"898287c2-9417-48b0-905a-280032b3fc2c"}
{"level":"info","ts":"2026-09-18T10:09:31.270Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"898287c2-9417-48b0-905a-280032b3fc2c"}
{"level":"info","ts":"2026-09-18T10:09:31.270Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"898287c2-9417-48b0-905a-280032b3fc2c"}
{"level":"info","ts":"2026-09-18T10:09:31.270Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"898287c2-9417-48b0-905a-280032b3fc2c","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:31.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"898287c2-9417-48b0-905a-280032b3fc2c","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:31.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"898287c2-9417-48b0-905a-280032b3fc2c","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:31.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"898287c2-9417-48b0-905a-280032b3fc2c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:31.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"898287c2-9417-48b0-905a-280032b3fc2c","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:31.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"898287c2-9417-48b0-905a-280032b3fc2c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:31.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"898287c2-9417-48b0-905a-280032b3fc2c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:31.270Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"898287c2-9417-48b0-905a-280032b3fc2c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:31.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"898287c2-9417-48b0-905a-280032b3fc2c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:31.270Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"898287c2-9417-48b0-905a-280032b3fc2c","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:09:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:09:31.270Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"898287c2-9417-48b0-905a-280032b3fc2c","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:09:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:09:31.270Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"898287c2-9417-48b0-905a-280032b3fc2c","name":"counter-auth-hw6hh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:09:31.288Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"c8f00ebe-5c54-454e-ad75-93d1abc93c27","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:31.288Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"c8f00ebe-5c54-454e-ad75-93d1abc93c27","pod":"counter-auth-hw6hh-head-ssjn9","ip":"10.133.0.48"}
{"level":"info","ts":"2026-09-18T10:09:31.288Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"c8f00ebe-5c54-454e-ad75-93d1abc93c27","pod":"counter-auth-hw6hh-small-group-worker-9mpg4","ip":"10.132.0.55"}
{"level":"info","ts":"2026-09-18T10:09:31.288Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"c8f00ebe-5c54-454e-ad75-93d1abc93c27","rayCluster":"counter-auth-hw6hh","podIPs":["10.133.0.48","10.132.0.55"]}
{"level":"info","ts":"2026-09-18T10:09:31.288Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7b695b09-5feb-4012-881f-a44fd93f3efc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:31.288Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"12961b7f-4738-41d3-bcca-0728fa05eb01","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:31.288Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7b695b09-5feb-4012-881f-a44fd93f3efc","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:31.288Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7b695b09-5feb-4012-881f-a44fd93f3efc","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:31.289Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7b695b09-5feb-4012-881f-a44fd93f3efc"}
{"level":"info","ts":"2026-09-18T10:09:31.289Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7b695b09-5feb-4012-881f-a44fd93f3efc","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:31.289Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7b695b09-5feb-4012-881f-a44fd93f3efc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:31.289Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7b695b09-5feb-4012-881f-a44fd93f3efc","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:09:31.314Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"978e28bf-75d5-4327-b2e3-d148e0e9057d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:31.314Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"978e28bf-75d5-4327-b2e3-d148e0e9057d","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:31.315Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"978e28bf-75d5-4327-b2e3-d148e0e9057d","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:31.315Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"978e28bf-75d5-4327-b2e3-d148e0e9057d"}
{"level":"info","ts":"2026-09-18T10:09:31.315Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"978e28bf-75d5-4327-b2e3-d148e0e9057d","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:31.315Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"978e28bf-75d5-4327-b2e3-d148e0e9057d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:31.373Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"12961b7f-4738-41d3-bcca-0728fa05eb01","name":"counter-auth-hw6hh-head"}
{"level":"info","ts":"2026-09-18T10:09:31.387Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"12961b7f-4738-41d3-bcca-0728fa05eb01","name":"counter-auth-hw6hh-workers"}
{"level":"info","ts":"2026-09-18T10:09:31.387Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"12961b7f-4738-41d3-bcca-0728fa05eb01","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:32.869Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"0d5e20d6-7e33-4b8c-9f83-8e6d762b6a62","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:32.869Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"0d5e20d6-7e33-4b8c-9f83-8e6d762b6a62","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:32.869Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"0d5e20d6-7e33-4b8c-9f83-8e6d762b6a62","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:32.869Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"0d5e20d6-7e33-4b8c-9f83-8e6d762b6a62"}
{"level":"info","ts":"2026-09-18T10:09:32.869Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"0d5e20d6-7e33-4b8c-9f83-8e6d762b6a62","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:32.869Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"0d5e20d6-7e33-4b8c-9f83-8e6d762b6a62","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:33.292Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"006c066a-6b71-423c-b1ee-122f690161fe"}
{"level":"info","ts":"2026-09-18T10:09:33.292Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"006c066a-6b71-423c-b1ee-122f690161fe"}
{"level":"info","ts":"2026-09-18T10:09:33.292Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"006c066a-6b71-423c-b1ee-122f690161fe"}
{"level":"info","ts":"2026-09-18T10:09:33.292Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"006c066a-6b71-423c-b1ee-122f690161fe","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:33.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"006c066a-6b71-423c-b1ee-122f690161fe","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:33.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"006c066a-6b71-423c-b1ee-122f690161fe","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:33.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"006c066a-6b71-423c-b1ee-122f690161fe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:33.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"006c066a-6b71-423c-b1ee-122f690161fe","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:33.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"006c066a-6b71-423c-b1ee-122f690161fe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:33.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"006c066a-6b71-423c-b1ee-122f690161fe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:33.292Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"006c066a-6b71-423c-b1ee-122f690161fe","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:33.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"006c066a-6b71-423c-b1ee-122f690161fe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:33.292Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"006c066a-6b71-423c-b1ee-122f690161fe","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:09:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:09:33.292Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"006c066a-6b71-423c-b1ee-122f690161fe","podName":"counter-auth-hw6hh-head-ssjn9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:09:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:09:33.292Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"006c066a-6b71-423c-b1ee-122f690161fe","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:09:33.292Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"006c066a-6b71-423c-b1ee-122f690161fe","seconds":300}
{"level":"info","ts":"2026-09-18T10:09:34.681Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"b9720d07-f90e-4293-bfb3-ed52174e18b2","namespace":"test-ns-ltx56","clusterName":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:09:34.681Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-lgrnm","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"submitter-timeout-lgrnm","reconcileID":"b9720d07-f90e-4293-bfb3-ed52174e18b2","namespace":"test-ns-ltx56","clusterName":"submitter-timeout-lgrnm"}
{"level":"info","ts":"2026-09-18T10:09:35.873Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"720edb46-4357-4252-88ad-30b9eae305ce","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:35.873Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"720edb46-4357-4252-88ad-30b9eae305ce","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:35.873Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"720edb46-4357-4252-88ad-30b9eae305ce","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:35.873Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"720edb46-4357-4252-88ad-30b9eae305ce"}
{"level":"info","ts":"2026-09-18T10:09:35.873Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"720edb46-4357-4252-88ad-30b9eae305ce","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:35.873Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"720edb46-4357-4252-88ad-30b9eae305ce","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:38.874Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e87f46c1-5ac0-4a21-98f8-dbc8f6bc02b2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:38.874Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e87f46c1-5ac0-4a21-98f8-dbc8f6bc02b2","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:38.874Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e87f46c1-5ac0-4a21-98f8-dbc8f6bc02b2","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:38.874Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e87f46c1-5ac0-4a21-98f8-dbc8f6bc02b2"}
{"level":"info","ts":"2026-09-18T10:09:38.874Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e87f46c1-5ac0-4a21-98f8-dbc8f6bc02b2","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:38.874Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e87f46c1-5ac0-4a21-98f8-dbc8f6bc02b2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:41.875Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"925cec12-1ccc-4d47-a519-608b5e187763","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:41.875Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"925cec12-1ccc-4d47-a519-608b5e187763","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:41.875Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"925cec12-1ccc-4d47-a519-608b5e187763","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:41.875Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"925cec12-1ccc-4d47-a519-608b5e187763"}
{"level":"info","ts":"2026-09-18T10:09:41.875Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"925cec12-1ccc-4d47-a519-608b5e187763","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:41.875Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"925cec12-1ccc-4d47-a519-608b5e187763","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:44.059Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"be174058-efe5-4dc3-96cc-e264fcadcbd9"}
{"level":"info","ts":"2026-09-18T10:09:44.059Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"be174058-efe5-4dc3-96cc-e264fcadcbd9"}
{"level":"info","ts":"2026-09-18T10:09:44.059Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"be174058-efe5-4dc3-96cc-e264fcadcbd9"}
{"level":"info","ts":"2026-09-18T10:09:44.059Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"be174058-efe5-4dc3-96cc-e264fcadcbd9","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:44.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"be174058-efe5-4dc3-96cc-e264fcadcbd9","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:44.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"be174058-efe5-4dc3-96cc-e264fcadcbd9","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:44.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"be174058-efe5-4dc3-96cc-e264fcadcbd9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:44.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"be174058-efe5-4dc3-96cc-e264fcadcbd9","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:44.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"be174058-efe5-4dc3-96cc-e264fcadcbd9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:44.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"be174058-efe5-4dc3-96cc-e264fcadcbd9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:44.059Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"be174058-efe5-4dc3-96cc-e264fcadcbd9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:44.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"be174058-efe5-4dc3-96cc-e264fcadcbd9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:44.059Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"be174058-efe5-4dc3-96cc-e264fcadcbd9","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:44.059Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"be174058-efe5-4dc3-96cc-e264fcadcbd9","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:44.059Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"be174058-efe5-4dc3-96cc-e264fcadcbd9","name":"counter-auth-hw6hh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:09:44.087Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e7deeac0-5e28-4d31-8845-848b2b5e5457","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:44.087Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e7deeac0-5e28-4d31-8845-848b2b5e5457","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:44.087Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e7deeac0-5e28-4d31-8845-848b2b5e5457","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:44.087Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e7deeac0-5e28-4d31-8845-848b2b5e5457"}
{"level":"info","ts":"2026-09-18T10:09:44.087Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e7deeac0-5e28-4d31-8845-848b2b5e5457","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:44.087Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e7deeac0-5e28-4d31-8845-848b2b5e5457","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:44.087Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e7deeac0-5e28-4d31-8845-848b2b5e5457","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:09:44.088Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"68bfb424-d0f8-460e-800a-1e9a77a250d8","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:44.088Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"db7a7859-7599-4b64-ba4e-3de185e1c24d","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:44.088Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"68bfb424-d0f8-460e-800a-1e9a77a250d8","pod":"counter-auth-hw6hh-head-ssjn9","ip":"10.133.0.48"}
{"level":"info","ts":"2026-09-18T10:09:44.088Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"68bfb424-d0f8-460e-800a-1e9a77a250d8","pod":"counter-auth-hw6hh-small-group-worker-9mpg4","ip":"10.132.0.55"}
{"level":"info","ts":"2026-09-18T10:09:44.088Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"68bfb424-d0f8-460e-800a-1e9a77a250d8","rayCluster":"counter-auth-hw6hh","podIPs":["10.133.0.48","10.132.0.55"]}
{"level":"info","ts":"2026-09-18T10:09:44.106Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"db7a7859-7599-4b64-ba4e-3de185e1c24d","name":"counter-auth-hw6hh-head"}
{"level":"info","ts":"2026-09-18T10:09:44.109Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9cc00c84-e798-4940-9fcb-80df128e9af8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:44.109Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9cc00c84-e798-4940-9fcb-80df128e9af8","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:44.109Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9cc00c84-e798-4940-9fcb-80df128e9af8","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:44.109Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9cc00c84-e798-4940-9fcb-80df128e9af8"}
{"level":"info","ts":"2026-09-18T10:09:44.109Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9cc00c84-e798-4940-9fcb-80df128e9af8","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:44.109Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9cc00c84-e798-4940-9fcb-80df128e9af8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:44.130Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"db7a7859-7599-4b64-ba4e-3de185e1c24d","name":"counter-auth-hw6hh-workers"}
{"level":"info","ts":"2026-09-18T10:09:44.130Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"db7a7859-7599-4b64-ba4e-3de185e1c24d","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:44.877Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c0d2ad9d-c03a-42b8-9735-85f1914d4ecd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:44.877Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c0d2ad9d-c03a-42b8-9735-85f1914d4ecd","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:44.877Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c0d2ad9d-c03a-42b8-9735-85f1914d4ecd","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:44.877Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c0d2ad9d-c03a-42b8-9735-85f1914d4ecd"}
{"level":"info","ts":"2026-09-18T10:09:44.877Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c0d2ad9d-c03a-42b8-9735-85f1914d4ecd","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:44.877Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c0d2ad9d-c03a-42b8-9735-85f1914d4ecd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:45.583Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"ffe58529-45cd-4bce-b0db-4a729d81280c","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:09:45.583Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"ffe58529-45cd-4bce-b0db-4a729d81280c","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:09:45.583Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"ffe58529-45cd-4bce-b0db-4a729d81280c","pod":"long-running-5qfl8-small-group-worker-tbqtj","ip":"10.132.0.47"}
{"level":"info","ts":"2026-09-18T10:09:45.583Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"ffe58529-45cd-4bce-b0db-4a729d81280c","rayCluster":"long-running-5qfl8","podIPs":["10.133.0.42","10.132.0.47"]}
{"level":"info","ts":"2026-09-18T10:09:46.085Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e53c1cd-53d6-4ddc-a3a5-3127fa7f2dfb"}
{"level":"info","ts":"2026-09-18T10:09:46.085Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e53c1cd-53d6-4ddc-a3a5-3127fa7f2dfb"}
{"level":"info","ts":"2026-09-18T10:09:46.085Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e53c1cd-53d6-4ddc-a3a5-3127fa7f2dfb"}
{"level":"info","ts":"2026-09-18T10:09:46.085Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e53c1cd-53d6-4ddc-a3a5-3127fa7f2dfb","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:09:46.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e53c1cd-53d6-4ddc-a3a5-3127fa7f2dfb","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:09:46.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e53c1cd-53d6-4ddc-a3a5-3127fa7f2dfb","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:46.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e53c1cd-53d6-4ddc-a3a5-3127fa7f2dfb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:09:46.086Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e53c1cd-53d6-4ddc-a3a5-3127fa7f2dfb","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:09:46.086Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e53c1cd-53d6-4ddc-a3a5-3127fa7f2dfb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:46.086Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e53c1cd-53d6-4ddc-a3a5-3127fa7f2dfb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:09:46.086Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e53c1cd-53d6-4ddc-a3a5-3127fa7f2dfb","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:09:46.086Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e53c1cd-53d6-4ddc-a3a5-3127fa7f2dfb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:09:46.086Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e53c1cd-53d6-4ddc-a3a5-3127fa7f2dfb","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:46.086Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e53c1cd-53d6-4ddc-a3a5-3127fa7f2dfb","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:09:46.086Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e53c1cd-53d6-4ddc-a3a5-3127fa7f2dfb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:09:46.086Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"2e53c1cd-53d6-4ddc-a3a5-3127fa7f2dfb","seconds":300}
{"level":"info","ts":"2026-09-18T10:09:47.878Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"24c4dcea-8e6b-4c95-a2fe-cb477bd62558","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:47.878Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"24c4dcea-8e6b-4c95-a2fe-cb477bd62558","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:47.878Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"24c4dcea-8e6b-4c95-a2fe-cb477bd62558","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:47.878Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"24c4dcea-8e6b-4c95-a2fe-cb477bd62558"}
{"level":"info","ts":"2026-09-18T10:09:47.878Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"24c4dcea-8e6b-4c95-a2fe-cb477bd62558","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:47.878Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"24c4dcea-8e6b-4c95-a2fe-cb477bd62558","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:50.879Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"2219d101-5992-476d-9def-162a24697e64","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:50.879Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"2219d101-5992-476d-9def-162a24697e64","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:50.879Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"2219d101-5992-476d-9def-162a24697e64","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:50.879Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"2219d101-5992-476d-9def-162a24697e64"}
{"level":"info","ts":"2026-09-18T10:09:50.879Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"2219d101-5992-476d-9def-162a24697e64","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:50.879Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"2219d101-5992-476d-9def-162a24697e64","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:52.369Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"afb6f085-d3b8-4a97-9a30-1770ffc1461b","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:52.369Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"afb6f085-d3b8-4a97-9a30-1770ffc1461b","pod":"counter-auth-hw6hh-head-ssjn9","ip":"10.133.0.48"}
{"level":"info","ts":"2026-09-18T10:09:52.369Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"afb6f085-d3b8-4a97-9a30-1770ffc1461b","pod":"counter-auth-hw6hh-small-group-worker-9mpg4","ip":"10.132.0.55"}
{"level":"info","ts":"2026-09-18T10:09:52.369Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"afb6f085-d3b8-4a97-9a30-1770ffc1461b","rayCluster":"counter-auth-hw6hh","podIPs":["10.133.0.48","10.132.0.55"]}
{"level":"info","ts":"2026-09-18T10:09:53.880Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c3da2e39-b520-4a5b-b481-b1e65b488037","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:53.880Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c3da2e39-b520-4a5b-b481-b1e65b488037","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:53.880Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c3da2e39-b520-4a5b-b481-b1e65b488037","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:53.880Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c3da2e39-b520-4a5b-b481-b1e65b488037"}
{"level":"info","ts":"2026-09-18T10:09:53.880Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c3da2e39-b520-4a5b-b481-b1e65b488037","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:53.880Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c3da2e39-b520-4a5b-b481-b1e65b488037","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:56.881Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"8ee0e4eb-5a62-43fb-9e48-af586a1df0a6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:56.881Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"8ee0e4eb-5a62-43fb-9e48-af586a1df0a6","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:56.881Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"8ee0e4eb-5a62-43fb-9e48-af586a1df0a6","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:56.881Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"8ee0e4eb-5a62-43fb-9e48-af586a1df0a6"}
{"level":"info","ts":"2026-09-18T10:09:56.881Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"8ee0e4eb-5a62-43fb-9e48-af586a1df0a6","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:56.881Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"8ee0e4eb-5a62-43fb-9e48-af586a1df0a6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:09:59.882Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"09d951ea-6358-4b8a-a44f-fa77e45ebb22","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:09:59.882Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"09d951ea-6358-4b8a-a44f-fa77e45ebb22","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:09:59.882Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"09d951ea-6358-4b8a-a44f-fa77e45ebb22","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:09:59.882Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"09d951ea-6358-4b8a-a44f-fa77e45ebb22"}
{"level":"info","ts":"2026-09-18T10:09:59.882Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"09d951ea-6358-4b8a-a44f-fa77e45ebb22","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:09:59.882Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"09d951ea-6358-4b8a-a44f-fa77e45ebb22","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:02.883Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c2411017-b42e-4cb7-ba4f-c00f8c25475c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:02.883Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c2411017-b42e-4cb7-ba4f-c00f8c25475c","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:02.883Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c2411017-b42e-4cb7-ba4f-c00f8c25475c","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:02.883Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c2411017-b42e-4cb7-ba4f-c00f8c25475c"}
{"level":"info","ts":"2026-09-18T10:10:02.883Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c2411017-b42e-4cb7-ba4f-c00f8c25475c","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:02.883Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c2411017-b42e-4cb7-ba4f-c00f8c25475c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:05.884Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"13e0d406-b9c6-4725-a33c-810f6894f8f0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:05.884Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"13e0d406-b9c6-4725-a33c-810f6894f8f0","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:05.884Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"13e0d406-b9c6-4725-a33c-810f6894f8f0","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:05.884Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"13e0d406-b9c6-4725-a33c-810f6894f8f0"}
{"level":"info","ts":"2026-09-18T10:10:05.884Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"13e0d406-b9c6-4725-a33c-810f6894f8f0","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:05.884Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"13e0d406-b9c6-4725-a33c-810f6894f8f0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:06.793Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"644db10a-1836-47c6-8c73-354880ab8a56","namespace":"test-ns-ltx56","clusterName":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:10:06.793Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-m6crj","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"cluster-status-update-m6crj","reconcileID":"644db10a-1836-47c6-8c73-354880ab8a56","namespace":"test-ns-ltx56","clusterName":"cluster-status-update-m6crj"}
{"level":"info","ts":"2026-09-18T10:10:08.884Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7c916f06-e25a-458c-ae1a-1540ee6df791","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:08.884Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7c916f06-e25a-458c-ae1a-1540ee6df791","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:08.884Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7c916f06-e25a-458c-ae1a-1540ee6df791","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:08.885Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7c916f06-e25a-458c-ae1a-1540ee6df791"}
{"level":"info","ts":"2026-09-18T10:10:08.885Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7c916f06-e25a-458c-ae1a-1540ee6df791","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:08.885Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7c916f06-e25a-458c-ae1a-1540ee6df791","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:11.886Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9aca1d73-27fb-4a70-b908-b6d558d5075e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:11.886Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9aca1d73-27fb-4a70-b908-b6d558d5075e","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:11.886Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9aca1d73-27fb-4a70-b908-b6d558d5075e","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:11.886Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9aca1d73-27fb-4a70-b908-b6d558d5075e"}
{"level":"info","ts":"2026-09-18T10:10:11.886Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9aca1d73-27fb-4a70-b908-b6d558d5075e","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:11.886Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9aca1d73-27fb-4a70-b908-b6d558d5075e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:14.887Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9cc9c7d3-783d-4c61-bddc-f14c3ada6791","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:14.887Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9cc9c7d3-783d-4c61-bddc-f14c3ada6791","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:14.887Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9cc9c7d3-783d-4c61-bddc-f14c3ada6791","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:14.887Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9cc9c7d3-783d-4c61-bddc-f14c3ada6791"}
{"level":"info","ts":"2026-09-18T10:10:14.887Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9cc9c7d3-783d-4c61-bddc-f14c3ada6791","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:14.887Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"9cc9c7d3-783d-4c61-bddc-f14c3ada6791","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:17.888Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d3e822c3-740a-4d8b-bfb7-65cfacab2877","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:17.888Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d3e822c3-740a-4d8b-bfb7-65cfacab2877","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:17.888Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d3e822c3-740a-4d8b-bfb7-65cfacab2877","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:17.889Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d3e822c3-740a-4d8b-bfb7-65cfacab2877"}
{"level":"info","ts":"2026-09-18T10:10:17.889Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d3e822c3-740a-4d8b-bfb7-65cfacab2877","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:17.889Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d3e822c3-740a-4d8b-bfb7-65cfacab2877","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:19.834Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"51dacb30-0c38-4427-8f06-0617cdc12895"}
{"level":"info","ts":"2026-09-18T10:10:19.834Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"51dacb30-0c38-4427-8f06-0617cdc12895"}
{"level":"info","ts":"2026-09-18T10:10:19.834Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"51dacb30-0c38-4427-8f06-0617cdc12895"}
{"level":"info","ts":"2026-09-18T10:10:19.834Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"51dacb30-0c38-4427-8f06-0617cdc12895","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:10:19.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"51dacb30-0c38-4427-8f06-0617cdc12895","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:10:19.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"51dacb30-0c38-4427-8f06-0617cdc12895","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:10:19.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"51dacb30-0c38-4427-8f06-0617cdc12895","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:10:19.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"51dacb30-0c38-4427-8f06-0617cdc12895","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:10:19.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"51dacb30-0c38-4427-8f06-0617cdc12895","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:10:19.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"51dacb30-0c38-4427-8f06-0617cdc12895","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:10:19.834Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"51dacb30-0c38-4427-8f06-0617cdc12895","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:10:19.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"51dacb30-0c38-4427-8f06-0617cdc12895","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:10:19.834Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"51dacb30-0c38-4427-8f06-0617cdc12895","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:10:19.834Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"51dacb30-0c38-4427-8f06-0617cdc12895","seconds":300}
{"level":"info","ts":"2026-09-18T10:10:20.890Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"832cb85f-3629-4167-8926-6c92c1ea92ca","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:20.890Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"832cb85f-3629-4167-8926-6c92c1ea92ca","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:20.890Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"832cb85f-3629-4167-8926-6c92c1ea92ca","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:20.890Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"832cb85f-3629-4167-8926-6c92c1ea92ca"}
{"level":"info","ts":"2026-09-18T10:10:20.890Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"832cb85f-3629-4167-8926-6c92c1ea92ca","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:20.890Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"832cb85f-3629-4167-8926-6c92c1ea92ca","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:23.891Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"08021295-2d6c-45c1-8dbb-fed5eae9f741","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:23.891Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"08021295-2d6c-45c1-8dbb-fed5eae9f741","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:23.891Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"08021295-2d6c-45c1-8dbb-fed5eae9f741","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:23.891Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"08021295-2d6c-45c1-8dbb-fed5eae9f741"}
{"level":"info","ts":"2026-09-18T10:10:23.891Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"08021295-2d6c-45c1-8dbb-fed5eae9f741","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:23.891Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"08021295-2d6c-45c1-8dbb-fed5eae9f741","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:26.892Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f3db3ed6-18cb-47d4-bf6c-b84e7d69e051","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:26.892Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f3db3ed6-18cb-47d4-bf6c-b84e7d69e051","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:26.892Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f3db3ed6-18cb-47d4-bf6c-b84e7d69e051","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:26.893Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f3db3ed6-18cb-47d4-bf6c-b84e7d69e051"}
{"level":"info","ts":"2026-09-18T10:10:26.893Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f3db3ed6-18cb-47d4-bf6c-b84e7d69e051","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:26.893Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f3db3ed6-18cb-47d4-bf6c-b84e7d69e051","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:29.893Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b3acd707-ddf2-4338-be8c-0fabe68b21b1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:29.893Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b3acd707-ddf2-4338-be8c-0fabe68b21b1","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:29.893Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b3acd707-ddf2-4338-be8c-0fabe68b21b1","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:29.893Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b3acd707-ddf2-4338-be8c-0fabe68b21b1"}
{"level":"info","ts":"2026-09-18T10:10:29.893Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b3acd707-ddf2-4338-be8c-0fabe68b21b1","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:29.893Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b3acd707-ddf2-4338-be8c-0fabe68b21b1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:32.894Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"03b9d664-a1de-45ae-9ca5-0e3a1f40d84d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:32.894Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"03b9d664-a1de-45ae-9ca5-0e3a1f40d84d","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:32.894Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"03b9d664-a1de-45ae-9ca5-0e3a1f40d84d","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:32.894Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"03b9d664-a1de-45ae-9ca5-0e3a1f40d84d"}
{"level":"info","ts":"2026-09-18T10:10:32.894Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"03b9d664-a1de-45ae-9ca5-0e3a1f40d84d","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:32.894Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"03b9d664-a1de-45ae-9ca5-0e3a1f40d84d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:35.895Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6402bfaa-820b-4be2-9c30-48655619228e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:35.895Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6402bfaa-820b-4be2-9c30-48655619228e","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:35.895Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6402bfaa-820b-4be2-9c30-48655619228e","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:35.895Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6402bfaa-820b-4be2-9c30-48655619228e"}
{"level":"info","ts":"2026-09-18T10:10:35.895Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6402bfaa-820b-4be2-9c30-48655619228e","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:35.895Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6402bfaa-820b-4be2-9c30-48655619228e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:38.896Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"dc166da9-ccbb-44c1-858a-973b4636c6b0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:38.896Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"dc166da9-ccbb-44c1-858a-973b4636c6b0","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:38.896Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"dc166da9-ccbb-44c1-858a-973b4636c6b0","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:38.896Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"dc166da9-ccbb-44c1-858a-973b4636c6b0"}
{"level":"info","ts":"2026-09-18T10:10:38.896Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"dc166da9-ccbb-44c1-858a-973b4636c6b0","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:38.896Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"dc166da9-ccbb-44c1-858a-973b4636c6b0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:40.387Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"7f5b93aa-fbb4-4725-a45f-b8f8e2085a29"}
{"level":"info","ts":"2026-09-18T10:10:40.387Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"7f5b93aa-fbb4-4725-a45f-b8f8e2085a29"}
{"level":"info","ts":"2026-09-18T10:10:40.387Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"7f5b93aa-fbb4-4725-a45f-b8f8e2085a29"}
{"level":"info","ts":"2026-09-18T10:10:40.387Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"7f5b93aa-fbb4-4725-a45f-b8f8e2085a29","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:10:40.388Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"7f5b93aa-fbb4-4725-a45f-b8f8e2085a29","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:10:40.388Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"7f5b93aa-fbb4-4725-a45f-b8f8e2085a29","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:10:40.388Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"7f5b93aa-fbb4-4725-a45f-b8f8e2085a29","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:10:40.388Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"7f5b93aa-fbb4-4725-a45f-b8f8e2085a29","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:10:40.388Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"7f5b93aa-fbb4-4725-a45f-b8f8e2085a29","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:10:40.388Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"7f5b93aa-fbb4-4725-a45f-b8f8e2085a29","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:10:40.388Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"7f5b93aa-fbb4-4725-a45f-b8f8e2085a29","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:10:40.388Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"7f5b93aa-fbb4-4725-a45f-b8f8e2085a29","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:10:40.388Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"7f5b93aa-fbb4-4725-a45f-b8f8e2085a29","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:10:40.388Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"7f5b93aa-fbb4-4725-a45f-b8f8e2085a29","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:10:40.388Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"7f5b93aa-fbb4-4725-a45f-b8f8e2085a29","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:10:40.388Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"7f5b93aa-fbb4-4725-a45f-b8f8e2085a29","seconds":300}
{"level":"info","ts":"2026-09-18T10:10:41.397Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"96d87d83-c07a-48e4-8c20-17dc3d0506e2"}
{"level":"info","ts":"2026-09-18T10:10:41.397Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"96d87d83-c07a-48e4-8c20-17dc3d0506e2"}
{"level":"info","ts":"2026-09-18T10:10:41.397Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"96d87d83-c07a-48e4-8c20-17dc3d0506e2"}
{"level":"info","ts":"2026-09-18T10:10:41.397Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"96d87d83-c07a-48e4-8c20-17dc3d0506e2","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:10:41.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"96d87d83-c07a-48e4-8c20-17dc3d0506e2","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:10:41.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"96d87d83-c07a-48e4-8c20-17dc3d0506e2","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:10:41.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"96d87d83-c07a-48e4-8c20-17dc3d0506e2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:10:41.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"96d87d83-c07a-48e4-8c20-17dc3d0506e2","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:10:41.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"96d87d83-c07a-48e4-8c20-17dc3d0506e2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:10:41.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"96d87d83-c07a-48e4-8c20-17dc3d0506e2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:10:41.398Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"96d87d83-c07a-48e4-8c20-17dc3d0506e2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:10:41.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"96d87d83-c07a-48e4-8c20-17dc3d0506e2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:10:41.398Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"96d87d83-c07a-48e4-8c20-17dc3d0506e2","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:10:41.398Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"96d87d83-c07a-48e4-8c20-17dc3d0506e2","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:10:41.398Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"96d87d83-c07a-48e4-8c20-17dc3d0506e2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:10:41.398Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"96d87d83-c07a-48e4-8c20-17dc3d0506e2","seconds":300}
{"level":"info","ts":"2026-09-18T10:10:41.897Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"65ed1996-9551-41d1-bcfb-03a5838fdf3d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:41.897Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"65ed1996-9551-41d1-bcfb-03a5838fdf3d","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:41.897Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"65ed1996-9551-41d1-bcfb-03a5838fdf3d","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:41.898Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"65ed1996-9551-41d1-bcfb-03a5838fdf3d"}
{"level":"info","ts":"2026-09-18T10:10:41.898Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"65ed1996-9551-41d1-bcfb-03a5838fdf3d","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:41.898Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"65ed1996-9551-41d1-bcfb-03a5838fdf3d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:44.899Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b21276fb-5a79-43aa-bfe7-3e7e186b4c63","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:44.899Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b21276fb-5a79-43aa-bfe7-3e7e186b4c63","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:44.899Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b21276fb-5a79-43aa-bfe7-3e7e186b4c63","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:44.899Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b21276fb-5a79-43aa-bfe7-3e7e186b4c63"}
{"level":"info","ts":"2026-09-18T10:10:44.899Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b21276fb-5a79-43aa-bfe7-3e7e186b4c63","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:44.899Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b21276fb-5a79-43aa-bfe7-3e7e186b4c63","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:45.584Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"efc43dde-1875-4772-940b-77aef31c9ce8","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:10:45.584Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"efc43dde-1875-4772-940b-77aef31c9ce8","pod":"long-running-5qfl8-small-group-worker-tbqtj","ip":"10.132.0.47"}
{"level":"info","ts":"2026-09-18T10:10:45.584Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"efc43dde-1875-4772-940b-77aef31c9ce8","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:10:45.584Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"efc43dde-1875-4772-940b-77aef31c9ce8","rayCluster":"long-running-5qfl8","podIPs":["10.132.0.47","10.133.0.42"]}
{"level":"info","ts":"2026-09-18T10:10:47.900Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"df151097-b305-4e94-88d0-edd640dc1924","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:47.900Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"df151097-b305-4e94-88d0-edd640dc1924","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:47.900Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"df151097-b305-4e94-88d0-edd640dc1924","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:47.900Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"df151097-b305-4e94-88d0-edd640dc1924"}
{"level":"info","ts":"2026-09-18T10:10:47.900Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"df151097-b305-4e94-88d0-edd640dc1924","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:47.900Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"df151097-b305-4e94-88d0-edd640dc1924","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:50.445Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"c363a8ca-5d1e-4dad-96d1-250863298ac9"}
{"level":"info","ts":"2026-09-18T10:10:50.445Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"c363a8ca-5d1e-4dad-96d1-250863298ac9"}
{"level":"info","ts":"2026-09-18T10:10:50.445Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"c363a8ca-5d1e-4dad-96d1-250863298ac9"}
{"level":"info","ts":"2026-09-18T10:10:50.445Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"c363a8ca-5d1e-4dad-96d1-250863298ac9","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:10:50.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"c363a8ca-5d1e-4dad-96d1-250863298ac9","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:10:50.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"c363a8ca-5d1e-4dad-96d1-250863298ac9","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:10:50.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"c363a8ca-5d1e-4dad-96d1-250863298ac9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:10:50.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"c363a8ca-5d1e-4dad-96d1-250863298ac9","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:10:50.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"c363a8ca-5d1e-4dad-96d1-250863298ac9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:10:50.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"c363a8ca-5d1e-4dad-96d1-250863298ac9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:10:50.445Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"c363a8ca-5d1e-4dad-96d1-250863298ac9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:10:50.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"c363a8ca-5d1e-4dad-96d1-250863298ac9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:10:50.445Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"c363a8ca-5d1e-4dad-96d1-250863298ac9","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:10:50.445Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"c363a8ca-5d1e-4dad-96d1-250863298ac9","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:10:50.445Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"c363a8ca-5d1e-4dad-96d1-250863298ac9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:10:50.445Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"c363a8ca-5d1e-4dad-96d1-250863298ac9","seconds":300}
{"level":"info","ts":"2026-09-18T10:10:50.901Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"529bd9e5-beae-4136-a2c4-1ae6ef86afc7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:50.901Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"529bd9e5-beae-4136-a2c4-1ae6ef86afc7","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:50.901Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"529bd9e5-beae-4136-a2c4-1ae6ef86afc7","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:50.901Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"529bd9e5-beae-4136-a2c4-1ae6ef86afc7"}
{"level":"info","ts":"2026-09-18T10:10:50.901Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"529bd9e5-beae-4136-a2c4-1ae6ef86afc7","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:50.901Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"529bd9e5-beae-4136-a2c4-1ae6ef86afc7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:51.451Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"fb4f7b1e-d18e-49b9-8107-6448481aa548"}
{"level":"info","ts":"2026-09-18T10:10:51.451Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"fb4f7b1e-d18e-49b9-8107-6448481aa548"}
{"level":"info","ts":"2026-09-18T10:10:51.451Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"fb4f7b1e-d18e-49b9-8107-6448481aa548"}
{"level":"info","ts":"2026-09-18T10:10:51.451Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"fb4f7b1e-d18e-49b9-8107-6448481aa548","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:10:51.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"fb4f7b1e-d18e-49b9-8107-6448481aa548","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:10:51.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"fb4f7b1e-d18e-49b9-8107-6448481aa548","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:10:51.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"fb4f7b1e-d18e-49b9-8107-6448481aa548","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:10:51.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"fb4f7b1e-d18e-49b9-8107-6448481aa548","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:10:51.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"fb4f7b1e-d18e-49b9-8107-6448481aa548","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:10:51.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"fb4f7b1e-d18e-49b9-8107-6448481aa548","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:10:51.452Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"fb4f7b1e-d18e-49b9-8107-6448481aa548","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:10:51.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"fb4f7b1e-d18e-49b9-8107-6448481aa548","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:10:51.452Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"fb4f7b1e-d18e-49b9-8107-6448481aa548","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:09:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:10:51.452Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"fb4f7b1e-d18e-49b9-8107-6448481aa548","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:09:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:10:51.452Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"fb4f7b1e-d18e-49b9-8107-6448481aa548","name":"counter-auth-hw6hh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:10:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:10:51.467Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"5ccf7fa8-a846-4b0c-8434-aa00538e29f9","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:51.467Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"654a7c07-753a-4f19-bf2b-1ec46265760b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:51.467Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"654a7c07-753a-4f19-bf2b-1ec46265760b","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:51.467Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"654a7c07-753a-4f19-bf2b-1ec46265760b","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:51.467Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"77f22f99-f502-4caf-bf5f-7e839eebcc18","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:51.467Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"654a7c07-753a-4f19-bf2b-1ec46265760b"}
{"level":"info","ts":"2026-09-18T10:10:51.467Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"654a7c07-753a-4f19-bf2b-1ec46265760b","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:51.467Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"654a7c07-753a-4f19-bf2b-1ec46265760b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:09:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:10:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:51.467Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"654a7c07-753a-4f19-bf2b-1ec46265760b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:10:51.467Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"77f22f99-f502-4caf-bf5f-7e839eebcc18","pod":"counter-auth-hw6hh-head-ssjn9","ip":"10.133.0.48"}
{"level":"info","ts":"2026-09-18T10:10:51.467Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"77f22f99-f502-4caf-bf5f-7e839eebcc18","pod":"counter-auth-hw6hh-small-group-worker-9mpg4","ip":"10.132.0.55"}
{"level":"info","ts":"2026-09-18T10:10:51.467Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"77f22f99-f502-4caf-bf5f-7e839eebcc18","rayCluster":"counter-auth-hw6hh","podIPs":["10.133.0.48","10.132.0.55"]}
{"level":"info","ts":"2026-09-18T10:10:51.484Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e3ba6e35-b660-4ba5-9a45-e23bc7e61005","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:51.484Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e3ba6e35-b660-4ba5-9a45-e23bc7e61005","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:51.484Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e3ba6e35-b660-4ba5-9a45-e23bc7e61005","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:51.484Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e3ba6e35-b660-4ba5-9a45-e23bc7e61005"}
{"level":"info","ts":"2026-09-18T10:10:51.484Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e3ba6e35-b660-4ba5-9a45-e23bc7e61005","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:51.484Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e3ba6e35-b660-4ba5-9a45-e23bc7e61005","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:10:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:10:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:51.490Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"5ccf7fa8-a846-4b0c-8434-aa00538e29f9","name":"counter-auth-hw6hh-head"}
{"level":"info","ts":"2026-09-18T10:10:51.503Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"5ccf7fa8-a846-4b0c-8434-aa00538e29f9","name":"counter-auth-hw6hh-workers"}
{"level":"info","ts":"2026-09-18T10:10:51.503Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"5ccf7fa8-a846-4b0c-8434-aa00538e29f9","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:52.370Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"4bc596f6-a665-468b-a65d-a3671b6e277f","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:52.370Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"4bc596f6-a665-468b-a65d-a3671b6e277f","pod":"counter-auth-hw6hh-head-ssjn9","ip":"10.133.0.48"}
{"level":"info","ts":"2026-09-18T10:10:52.370Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"4bc596f6-a665-468b-a65d-a3671b6e277f","pod":"counter-auth-hw6hh-small-group-worker-9mpg4","ip":"10.132.0.55"}
{"level":"info","ts":"2026-09-18T10:10:52.370Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"4bc596f6-a665-468b-a65d-a3671b6e277f","rayCluster":"counter-auth-hw6hh","podIPs":["10.133.0.48","10.132.0.55"]}
{"level":"info","ts":"2026-09-18T10:10:53.467Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6c893c3a-59ce-4ae1-98ab-7ef35c4f7b6a"}
{"level":"info","ts":"2026-09-18T10:10:53.467Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6c893c3a-59ce-4ae1-98ab-7ef35c4f7b6a"}
{"level":"info","ts":"2026-09-18T10:10:53.467Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6c893c3a-59ce-4ae1-98ab-7ef35c4f7b6a"}
{"level":"info","ts":"2026-09-18T10:10:53.467Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6c893c3a-59ce-4ae1-98ab-7ef35c4f7b6a","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:10:53.467Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6c893c3a-59ce-4ae1-98ab-7ef35c4f7b6a","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:10:53.467Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6c893c3a-59ce-4ae1-98ab-7ef35c4f7b6a","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:10:53.467Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6c893c3a-59ce-4ae1-98ab-7ef35c4f7b6a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:10:53.467Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6c893c3a-59ce-4ae1-98ab-7ef35c4f7b6a","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:10:53.467Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6c893c3a-59ce-4ae1-98ab-7ef35c4f7b6a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:10:53.467Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6c893c3a-59ce-4ae1-98ab-7ef35c4f7b6a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:10:53.467Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6c893c3a-59ce-4ae1-98ab-7ef35c4f7b6a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:10:53.467Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6c893c3a-59ce-4ae1-98ab-7ef35c4f7b6a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:10:53.467Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6c893c3a-59ce-4ae1-98ab-7ef35c4f7b6a","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:09:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:10:53.467Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6c893c3a-59ce-4ae1-98ab-7ef35c4f7b6a","podName":"counter-auth-hw6hh-small-group-worker-9mpg4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:09:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:10:53.467Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6c893c3a-59ce-4ae1-98ab-7ef35c4f7b6a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:10:53.467Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6c893c3a-59ce-4ae1-98ab-7ef35c4f7b6a","seconds":300}
{"level":"info","ts":"2026-09-18T10:10:53.902Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"447d966e-8b2f-4b3e-af1f-de81e90ec736","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:53.902Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"447d966e-8b2f-4b3e-af1f-de81e90ec736","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:53.902Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"447d966e-8b2f-4b3e-af1f-de81e90ec736","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:53.902Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"447d966e-8b2f-4b3e-af1f-de81e90ec736"}
{"level":"info","ts":"2026-09-18T10:10:53.902Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"447d966e-8b2f-4b3e-af1f-de81e90ec736","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:53.902Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"447d966e-8b2f-4b3e-af1f-de81e90ec736","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:10:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:10:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:56.903Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"158653ec-a62e-4feb-9cc3-2e897bda0447","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:56.903Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"158653ec-a62e-4feb-9cc3-2e897bda0447","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:56.903Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"158653ec-a62e-4feb-9cc3-2e897bda0447","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:56.903Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"158653ec-a62e-4feb-9cc3-2e897bda0447"}
{"level":"info","ts":"2026-09-18T10:10:56.903Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"158653ec-a62e-4feb-9cc3-2e897bda0447","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:56.903Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"158653ec-a62e-4feb-9cc3-2e897bda0447","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:10:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:10:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:10:59.904Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"3ed67430-3a0a-47c4-8537-44ebf1cf6d25","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:10:59.904Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"3ed67430-3a0a-47c4-8537-44ebf1cf6d25","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:10:59.905Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"3ed67430-3a0a-47c4-8537-44ebf1cf6d25","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:10:59.905Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"3ed67430-3a0a-47c4-8537-44ebf1cf6d25"}
{"level":"info","ts":"2026-09-18T10:10:59.905Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"3ed67430-3a0a-47c4-8537-44ebf1cf6d25","RayCluster":"counter-auth-hw6hh","State":""}
{"level":"info","ts":"2026-09-18T10:10:59.905Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"3ed67430-3a0a-47c4-8537-44ebf1cf6d25","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:10:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:10:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:02.479Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6bdc4f0d-f0d0-4ec3-b760-dffee1ae259e"}
{"level":"info","ts":"2026-09-18T10:11:02.479Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6bdc4f0d-f0d0-4ec3-b760-dffee1ae259e"}
{"level":"info","ts":"2026-09-18T10:11:02.479Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6bdc4f0d-f0d0-4ec3-b760-dffee1ae259e"}
{"level":"info","ts":"2026-09-18T10:11:02.479Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6bdc4f0d-f0d0-4ec3-b760-dffee1ae259e","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:02.479Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6bdc4f0d-f0d0-4ec3-b760-dffee1ae259e","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:02.479Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6bdc4f0d-f0d0-4ec3-b760-dffee1ae259e","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:02.479Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6bdc4f0d-f0d0-4ec3-b760-dffee1ae259e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:02.479Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6bdc4f0d-f0d0-4ec3-b760-dffee1ae259e","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:02.479Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6bdc4f0d-f0d0-4ec3-b760-dffee1ae259e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:02.479Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6bdc4f0d-f0d0-4ec3-b760-dffee1ae259e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:02.479Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6bdc4f0d-f0d0-4ec3-b760-dffee1ae259e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:02.479Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6bdc4f0d-f0d0-4ec3-b760-dffee1ae259e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:02.479Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6bdc4f0d-f0d0-4ec3-b760-dffee1ae259e","name":"counter-auth-hw6hh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:10:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:11:02.494Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"aef9460e-0ecd-436f-b009-bad3d47c286e","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:02.494Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"a819f8cd-b942-4328-8683-e6e8245a6519","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:02.494Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"a819f8cd-b942-4328-8683-e6e8245a6519","pod":"counter-auth-hw6hh-head-ssjn9","ip":"10.133.0.48"}
{"level":"info","ts":"2026-09-18T10:11:02.494Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"a819f8cd-b942-4328-8683-e6e8245a6519","pod":"counter-auth-hw6hh-small-group-worker-9mpg4","ip":"10.132.0.55"}
{"level":"info","ts":"2026-09-18T10:11:02.494Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"a819f8cd-b942-4328-8683-e6e8245a6519","rayCluster":"counter-auth-hw6hh","podIPs":["10.133.0.48","10.132.0.55"]}
{"level":"info","ts":"2026-09-18T10:11:02.494Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6aea27cc-e304-4bd7-837d-ce06b6a9198a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:02.494Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6aea27cc-e304-4bd7-837d-ce06b6a9198a","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:02.494Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6aea27cc-e304-4bd7-837d-ce06b6a9198a","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:02.495Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6aea27cc-e304-4bd7-837d-ce06b6a9198a"}
{"level":"info","ts":"2026-09-18T10:11:02.519Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6aea27cc-e304-4bd7-837d-ce06b6a9198a","Kubernetes Job":"counter-auth"}
{"level":"info","ts":"2026-09-18T10:11:02.519Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6aea27cc-e304-4bd7-837d-ce06b6a9198a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:10:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:10:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:02.519Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6aea27cc-e304-4bd7-837d-ce06b6a9198a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:11:02.524Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"aef9460e-0ecd-436f-b009-bad3d47c286e","name":"counter-auth-hw6hh-head"}
{"level":"info","ts":"2026-09-18T10:11:02.536Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"aef9460e-0ecd-436f-b009-bad3d47c286e","name":"counter-auth-hw6hh-workers"}
{"level":"info","ts":"2026-09-18T10:11:02.536Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"aef9460e-0ecd-436f-b009-bad3d47c286e","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:02.536Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"ec4a85a3-cff4-42a9-84ca-ddc125e88d87","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:02.536Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"ec4a85a3-cff4-42a9-84ca-ddc125e88d87","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:02.536Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"ec4a85a3-cff4-42a9-84ca-ddc125e88d87","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:02.536Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"ec4a85a3-cff4-42a9-84ca-ddc125e88d87"}
{"level":"info","ts":"2026-09-18T10:11:02.536Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"ec4a85a3-cff4-42a9-84ca-ddc125e88d87","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:11:02.573Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"ec4a85a3-cff4-42a9-84ca-ddc125e88d87","JobId":"counter-auth-5hdbw","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:02.573Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"ec4a85a3-cff4-42a9-84ca-ddc125e88d87"}
{"level":"error","ts":"2026-09-18T10:11:02.573Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"ec4a85a3-cff4-42a9-84ca-ddc125e88d87","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:02.573Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a1e8d82c-720c-438e-a834-b72095aac5c4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:02.573Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a1e8d82c-720c-438e-a834-b72095aac5c4","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:02.573Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a1e8d82c-720c-438e-a834-b72095aac5c4","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:02.573Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a1e8d82c-720c-438e-a834-b72095aac5c4"}
{"level":"info","ts":"2026-09-18T10:11:02.573Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a1e8d82c-720c-438e-a834-b72095aac5c4","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:11:02.578Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a1e8d82c-720c-438e-a834-b72095aac5c4","JobId":"counter-auth-5hdbw","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:02.578Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a1e8d82c-720c-438e-a834-b72095aac5c4"}
{"level":"error","ts":"2026-09-18T10:11:02.578Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a1e8d82c-720c-438e-a834-b72095aac5c4","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:02.579Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f355c94d-336c-4e95-9e46-20b0a92dc202","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:02.579Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f355c94d-336c-4e95-9e46-20b0a92dc202","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:02.579Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f355c94d-336c-4e95-9e46-20b0a92dc202","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:02.579Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f355c94d-336c-4e95-9e46-20b0a92dc202"}
{"level":"info","ts":"2026-09-18T10:11:02.579Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f355c94d-336c-4e95-9e46-20b0a92dc202","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:11:02.584Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f355c94d-336c-4e95-9e46-20b0a92dc202","JobId":"counter-auth-5hdbw","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:02.584Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f355c94d-336c-4e95-9e46-20b0a92dc202"}
{"level":"error","ts":"2026-09-18T10:11:02.584Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"f355c94d-336c-4e95-9e46-20b0a92dc202","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:02.604Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"22edf6f6-82eb-4271-b696-10ec4c8536aa","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:02.604Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"22edf6f6-82eb-4271-b696-10ec4c8536aa","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:02.604Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"22edf6f6-82eb-4271-b696-10ec4c8536aa","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:02.604Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"22edf6f6-82eb-4271-b696-10ec4c8536aa"}
{"level":"info","ts":"2026-09-18T10:11:02.604Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"22edf6f6-82eb-4271-b696-10ec4c8536aa","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:11:02.609Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"22edf6f6-82eb-4271-b696-10ec4c8536aa","JobId":"counter-auth-5hdbw","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:02.609Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"22edf6f6-82eb-4271-b696-10ec4c8536aa"}
{"level":"error","ts":"2026-09-18T10:11:02.609Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"22edf6f6-82eb-4271-b696-10ec4c8536aa","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:02.649Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c8289aa3-12b8-4ca7-ac98-5d891f9c858d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:02.649Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c8289aa3-12b8-4ca7-ac98-5d891f9c858d","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:02.649Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c8289aa3-12b8-4ca7-ac98-5d891f9c858d","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:02.649Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c8289aa3-12b8-4ca7-ac98-5d891f9c858d"}
{"level":"info","ts":"2026-09-18T10:11:02.649Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c8289aa3-12b8-4ca7-ac98-5d891f9c858d","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:11:02.653Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c8289aa3-12b8-4ca7-ac98-5d891f9c858d","JobId":"counter-auth-5hdbw","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:02.653Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c8289aa3-12b8-4ca7-ac98-5d891f9c858d"}
{"level":"error","ts":"2026-09-18T10:11:02.653Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"c8289aa3-12b8-4ca7-ac98-5d891f9c858d","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:02.734Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"addfea8a-0ea4-4e3e-80ca-9620260cc168","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:02.734Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"addfea8a-0ea4-4e3e-80ca-9620260cc168","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:02.734Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"addfea8a-0ea4-4e3e-80ca-9620260cc168","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:02.734Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"addfea8a-0ea4-4e3e-80ca-9620260cc168"}
{"level":"info","ts":"2026-09-18T10:11:02.734Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"addfea8a-0ea4-4e3e-80ca-9620260cc168","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:11:02.739Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"addfea8a-0ea4-4e3e-80ca-9620260cc168","JobId":"counter-auth-5hdbw","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:02.739Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"addfea8a-0ea4-4e3e-80ca-9620260cc168"}
{"level":"error","ts":"2026-09-18T10:11:02.739Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"addfea8a-0ea4-4e3e-80ca-9620260cc168","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:02.900Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"45169deb-5d82-4512-8b86-55ec0abb3a91","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:02.900Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"45169deb-5d82-4512-8b86-55ec0abb3a91","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:02.900Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"45169deb-5d82-4512-8b86-55ec0abb3a91","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:02.900Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"45169deb-5d82-4512-8b86-55ec0abb3a91"}
{"level":"info","ts":"2026-09-18T10:11:02.900Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"45169deb-5d82-4512-8b86-55ec0abb3a91","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:11:02.904Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"45169deb-5d82-4512-8b86-55ec0abb3a91","JobId":"counter-auth-5hdbw","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:02.905Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"45169deb-5d82-4512-8b86-55ec0abb3a91"}
{"level":"error","ts":"2026-09-18T10:11:02.905Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"45169deb-5d82-4512-8b86-55ec0abb3a91","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:03.225Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e2a6b89d-aa05-4cac-b841-6d0ba86e14ac","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:03.225Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e2a6b89d-aa05-4cac-b841-6d0ba86e14ac","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:03.225Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e2a6b89d-aa05-4cac-b841-6d0ba86e14ac","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:03.226Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e2a6b89d-aa05-4cac-b841-6d0ba86e14ac"}
{"level":"info","ts":"2026-09-18T10:11:03.226Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e2a6b89d-aa05-4cac-b841-6d0ba86e14ac","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:11:03.230Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e2a6b89d-aa05-4cac-b841-6d0ba86e14ac","JobId":"counter-auth-5hdbw","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:03.230Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e2a6b89d-aa05-4cac-b841-6d0ba86e14ac"}
{"level":"error","ts":"2026-09-18T10:11:03.230Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"e2a6b89d-aa05-4cac-b841-6d0ba86e14ac","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:03.557Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"41c83a46-77f2-4206-969b-e0a1c94d6376","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:03.557Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"41c83a46-77f2-4206-969b-e0a1c94d6376","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:03.557Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"41c83a46-77f2-4206-969b-e0a1c94d6376","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:03.557Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"41c83a46-77f2-4206-969b-e0a1c94d6376"}
{"level":"info","ts":"2026-09-18T10:11:03.557Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"41c83a46-77f2-4206-969b-e0a1c94d6376","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:11:03.562Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"41c83a46-77f2-4206-969b-e0a1c94d6376","JobId":"counter-auth-5hdbw","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:03.562Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"41c83a46-77f2-4206-969b-e0a1c94d6376"}
{"level":"error","ts":"2026-09-18T10:11:03.562Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"41c83a46-77f2-4206-969b-e0a1c94d6376","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:03.871Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6287f104-b9f2-4f58-8919-13ce12ed1759","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:03.871Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6287f104-b9f2-4f58-8919-13ce12ed1759","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:03.871Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6287f104-b9f2-4f58-8919-13ce12ed1759","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:03.871Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6287f104-b9f2-4f58-8919-13ce12ed1759"}
{"level":"info","ts":"2026-09-18T10:11:03.871Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6287f104-b9f2-4f58-8919-13ce12ed1759","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:11:03.876Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6287f104-b9f2-4f58-8919-13ce12ed1759","JobId":"counter-auth-5hdbw","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:03.876Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6287f104-b9f2-4f58-8919-13ce12ed1759"}
{"level":"error","ts":"2026-09-18T10:11:03.876Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"6287f104-b9f2-4f58-8919-13ce12ed1759","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:04.495Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5a015862-7771-4546-903c-75268454aed3"}
{"level":"info","ts":"2026-09-18T10:11:04.495Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5a015862-7771-4546-903c-75268454aed3"}
{"level":"info","ts":"2026-09-18T10:11:04.495Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5a015862-7771-4546-903c-75268454aed3"}
{"level":"info","ts":"2026-09-18T10:11:04.495Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5a015862-7771-4546-903c-75268454aed3","1 head service found":"counter-auth-hw6hh-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:04.495Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5a015862-7771-4546-903c-75268454aed3","Found 1 head Pod":"counter-auth-hw6hh-head-ssjn9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:04.495Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5a015862-7771-4546-903c-75268454aed3","head Pod":"counter-auth-hw6hh-head-ssjn9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-hw6hh-head-ssjn9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:04.495Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5a015862-7771-4546-903c-75268454aed3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:04.495Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5a015862-7771-4546-903c-75268454aed3","worker Pod":"counter-auth-hw6hh-small-group-worker-9mpg4","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-hw6hh-small-group-worker-9mpg4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:04.495Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5a015862-7771-4546-903c-75268454aed3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:04.495Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5a015862-7771-4546-903c-75268454aed3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:04.495Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5a015862-7771-4546-903c-75268454aed3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:04.495Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5a015862-7771-4546-903c-75268454aed3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:04.495Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5a015862-7771-4546-903c-75268454aed3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:11:04.495Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"5a015862-7771-4546-903c-75268454aed3","seconds":300}
{"level":"info","ts":"2026-09-18T10:11:06.437Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"ae7bf290-b03f-414f-ba56-3543dae6a8d6","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:06.437Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"ae7bf290-b03f-414f-ba56-3543dae6a8d6","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:06.437Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"ae7bf290-b03f-414f-ba56-3543dae6a8d6","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:06.438Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"ae7bf290-b03f-414f-ba56-3543dae6a8d6"}
{"level":"info","ts":"2026-09-18T10:11:06.438Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"ae7bf290-b03f-414f-ba56-3543dae6a8d6","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"error","ts":"2026-09-18T10:11:06.442Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"ae7bf290-b03f-414f-ba56-3543dae6a8d6","JobId":"counter-auth-5hdbw","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:330\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:06.442Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"ae7bf290-b03f-414f-ba56-3543dae6a8d6"}
{"level":"error","ts":"2026-09-18T10:11:06.442Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"ae7bf290-b03f-414f-ba56-3543dae6a8d6","error":"Job counter-auth-5hdbw does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:11.563Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"4a34113a-8361-4473-803f-5990c5c3a40c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:11.563Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"4a34113a-8361-4473-803f-5990c5c3a40c","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:11.563Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"4a34113a-8361-4473-803f-5990c5c3a40c","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:11.563Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"4a34113a-8361-4473-803f-5990c5c3a40c"}
{"level":"info","ts":"2026-09-18T10:11:11.563Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"4a34113a-8361-4473-803f-5990c5c3a40c","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:11:11.569Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"4a34113a-8361-4473-803f-5990c5c3a40c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:10:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:11.569Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"4a34113a-8361-4473-803f-5990c5c3a40c","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:11:11.586Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a5495aad-ed3f-44ae-a815-c1972fde8e7b","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:11.586Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a5495aad-ed3f-44ae-a815-c1972fde8e7b","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:11.586Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a5495aad-ed3f-44ae-a815-c1972fde8e7b","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:11.586Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a5495aad-ed3f-44ae-a815-c1972fde8e7b"}
{"level":"info","ts":"2026-09-18T10:11:11.586Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a5495aad-ed3f-44ae-a815-c1972fde8e7b","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:11:11.589Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"a5495aad-ed3f-44ae-a815-c1972fde8e7b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:14.585Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"872eed00-6afc-47a4-812f-b605bc4355e5","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:14.585Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"872eed00-6afc-47a4-812f-b605bc4355e5","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:14.585Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"872eed00-6afc-47a4-812f-b605bc4355e5","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:14.585Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"872eed00-6afc-47a4-812f-b605bc4355e5"}
{"level":"info","ts":"2026-09-18T10:11:14.585Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"872eed00-6afc-47a4-812f-b605bc4355e5","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:11:14.588Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"872eed00-6afc-47a4-812f-b605bc4355e5","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z","endTime":"2026-09-18T10:11:14Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:14.588Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"872eed00-6afc-47a4-812f-b605bc4355e5","old JobStatus":"PENDING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-09-18T10:11:14.608Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b5646994-008e-4929-ae53-b9a533511317","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:14.608Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b5646994-008e-4929-ae53-b9a533511317","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:14.608Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b5646994-008e-4929-ae53-b9a533511317","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:14.609Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b5646994-008e-4929-ae53-b9a533511317"}
{"level":"info","ts":"2026-09-18T10:11:14.609Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b5646994-008e-4929-ae53-b9a533511317","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:11:14.612Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"b5646994-008e-4929-ae53-b9a533511317","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z","endTime":"2026-09-18T10:11:14Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z","endTime":"2026-09-18T10:11:14Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:17.607Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d9f4517a-e7c0-4f9b-b8f4-f55e5311dbfb","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:17.607Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d9f4517a-e7c0-4f9b-b8f4-f55e5311dbfb","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:17.607Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d9f4517a-e7c0-4f9b-b8f4-f55e5311dbfb","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:17.607Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d9f4517a-e7c0-4f9b-b8f4-f55e5311dbfb"}
{"level":"info","ts":"2026-09-18T10:11:17.607Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d9f4517a-e7c0-4f9b-b8f4-f55e5311dbfb","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:11:17.610Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d9f4517a-e7c0-4f9b-b8f4-f55e5311dbfb","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z","endTime":"2026-09-18T10:11:14Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z","endTime":"2026-09-18T10:11:14Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:20.611Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"8f611dc8-e17f-464c-a80a-82901e3c1d94","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:20.611Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"8f611dc8-e17f-464c-a80a-82901e3c1d94","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:20.611Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"8f611dc8-e17f-464c-a80a-82901e3c1d94","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:20.611Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"8f611dc8-e17f-464c-a80a-82901e3c1d94"}
{"level":"info","ts":"2026-09-18T10:11:20.612Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"8f611dc8-e17f-464c-a80a-82901e3c1d94","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:11:20.615Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"8f611dc8-e17f-464c-a80a-82901e3c1d94","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z","endTime":"2026-09-18T10:11:14Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z","endTime":"2026-09-18T10:11:14Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:21.601Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"3f839901-9b01-467a-be74-114a361bc797","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:21.601Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"3f839901-9b01-467a-be74-114a361bc797","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:21.601Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"3f839901-9b01-467a-be74-114a361bc797","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:21.601Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"3f839901-9b01-467a-be74-114a361bc797"}
{"level":"info","ts":"2026-09-18T10:11:21.601Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"3f839901-9b01-467a-be74-114a361bc797","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:11:21.604Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"3f839901-9b01-467a-be74-114a361bc797","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z","endTime":"2026-09-18T10:11:14Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z","endTime":"2026-09-18T10:11:14Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:22.613Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"cdec5f4a-4fcd-4e67-914d-4b40fa6e26ff","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:22.613Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"cdec5f4a-4fcd-4e67-914d-4b40fa6e26ff","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:22.613Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"cdec5f4a-4fcd-4e67-914d-4b40fa6e26ff","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:22.613Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"cdec5f4a-4fcd-4e67-914d-4b40fa6e26ff"}
{"level":"info","ts":"2026-09-18T10:11:22.613Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"cdec5f4a-4fcd-4e67-914d-4b40fa6e26ff","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:11:22.616Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"cdec5f4a-4fcd-4e67-914d-4b40fa6e26ff","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z","endTime":"2026-09-18T10:11:14Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z","endTime":"2026-09-18T10:11:14Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:22.642Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"1105af80-1505-403e-bb14-775f3b19a2ad","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:22.642Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"1105af80-1505-403e-bb14-775f3b19a2ad","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:22.642Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"1105af80-1505-403e-bb14-775f3b19a2ad","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:22.642Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"1105af80-1505-403e-bb14-775f3b19a2ad"}
{"level":"info","ts":"2026-09-18T10:11:22.642Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"1105af80-1505-403e-bb14-775f3b19a2ad","rayCluster":"counter-auth-hw6hh","serviceType":"headService"}
{"level":"info","ts":"2026-09-18T10:11:22.644Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"1105af80-1505-403e-bb14-775f3b19a2ad","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z","endTime":"2026-09-18T10:11:14Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-09-18T10:09:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-09-18T10:11:10Z","endTime":"2026-09-18T10:11:14Z"},"jobId":"counter-auth-5hdbw","rayClusterName":"counter-auth-hw6hh","dashboardURL":"counter-auth-hw6hh-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-09-18T10:09:21Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:02Z","stateTransitionTimes":{"ready":"2026-09-18T10:11:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.48","serviceIP":"10.133.0.48","podName":"counter-auth-hw6hh-head-ssjn9","serviceName":"counter-auth-hw6hh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:09:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:11:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:09:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:09:22Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:22.644Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"1105af80-1505-403e-bb14-775f3b19a2ad","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-09-18T10:11:22.666Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7edec8fb-9fc6-4a26-ac94-605bb881e0c7","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:22.666Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7edec8fb-9fc6-4a26-ac94-605bb881e0c7","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:11:22 +0000 UTC","Now":"2026-09-18T10:11:22.666Z","ShutdownTime":"2026-09-18T10:11:22.000Z"}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7edec8fb-9fc6-4a26-ac94-605bb881e0c7","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7edec8fb-9fc6-4a26-ac94-605bb881e0c7","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"7edec8fb-9fc6-4a26-ac94-605bb881e0c7","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"reconcileID":"6f977b2c-d622-4dac-b004-c52fc544d42a"}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"51d809e7-ed0b-4b9b-a767-0c74501df295","rayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"6f5cae92-3dba-4158-9eaa-0edb3a385dcf"}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"f281f74f-c3ed-4937-8339-8d39f2e2621d","namespacedName":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"f281f74f-c3ed-4937-8339-8d39f2e2621d","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"f281f74f-c3ed-4937-8339-8d39f2e2621d","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"51d809e7-ed0b-4b9b-a767-0c74501df295","pod":"counter-auth-hw6hh-head-ssjn9","ip":"10.133.0.48"}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"51d809e7-ed0b-4b9b-a767-0c74501df295","pod":"counter-auth-hw6hh-small-group-worker-9mpg4","ip":"10.132.0.55"}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"f281f74f-c3ed-4937-8339-8d39f2e2621d","namespace":"test-ns-ltx56","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"51d809e7-ed0b-4b9b-a767-0c74501df295","rayCluster":"counter-auth-hw6hh","podIPs":["10.133.0.48","10.132.0.55"]}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"f281f74f-c3ed-4937-8339-8d39f2e2621d","name":"kuberay-gateway-access","namespace":"test-ns-ltx56","remainingClusters":1}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"95b94f9e-f81d-4fa3-b782-78ac9437563a","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"95b94f9e-f81d-4fa3-b782-78ac9437563a","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:11:22 +0000 UTC","Now":"2026-09-18T10:11:22.682Z","ShutdownTime":"2026-09-18T10:11:22.000Z"}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"95b94f9e-f81d-4fa3-b782-78ac9437563a","RayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"95b94f9e-f81d-4fa3-b782-78ac9437563a","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T10:11:22.682Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"95b94f9e-f81d-4fa3-b782-78ac9437563a","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:22.689Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"f281f74f-c3ed-4937-8339-8d39f2e2621d","name":"test-ns-ltx56-counter-auth-hw6hh","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T10:11:22.700Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"f281f74f-c3ed-4937-8339-8d39f2e2621d","configMap":"kube-rbac-proxy-config-counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:22.711Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"f281f74f-c3ed-4937-8339-8d39f2e2621d","serviceAccount":"counter-auth-hw6hh-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:11:22.740Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:22.740Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:11:22.763Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"5e7d7311-a4ac-4301-ac3a-0e617974e5f7"}
{"level":"info","ts":"2026-09-18T10:11:22.763Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"d97c3680-1818-4628-9bc5-6bdc9f6fd54d","namespace":"test-ns-ltx56","clusterName":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:22.763Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d7620b61-e5fd-4d6b-a6f9-934d443bc547","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:22.764Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d7620b61-e5fd-4d6b-a6f9-934d443bc547","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:11:22 +0000 UTC","Now":"2026-09-18T10:11:22.764Z","ShutdownTime":"2026-09-18T10:11:22.000Z"}
{"level":"info","ts":"2026-09-18T10:11:22.764Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d7620b61-e5fd-4d6b-a6f9-934d443bc547","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:22.764Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d7620b61-e5fd-4d6b-a6f9-934d443bc547","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T10:11:22.764Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d7620b61-e5fd-4d6b-a6f9-934d443bc547","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:22.764Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"f281f74f-c3ed-4937-8339-8d39f2e2621d","cluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:22.764Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"182bcd05-1df3-4cae-9fc0-4fa98fc1cd77","namespacedName":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:22.764Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"182bcd05-1df3-4cae-9fc0-4fa98fc1cd77"}
{"level":"info","ts":"2026-09-18T10:11:22.764Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"182bcd05-1df3-4cae-9fc0-4fa98fc1cd77","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:11:22.764Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"182bcd05-1df3-4cae-9fc0-4fa98fc1cd77","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:11:22.765Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"290ac3cf-6f3f-4d80-80cb-91e95e638d81","namespacedName":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:22.765Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"290ac3cf-6f3f-4d80-80cb-91e95e638d81"}
{"level":"info","ts":"2026-09-18T10:11:22.766Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"290ac3cf-6f3f-4d80-80cb-91e95e638d81","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:11:22.766Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"290ac3cf-6f3f-4d80-80cb-91e95e638d81","namespace":"test-ns-ltx56","authClusters":1}
{"level":"error","ts":"2026-09-18T10:11:22.799Z","msg":"Failed to delete CA issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"d97c3680-1818-4628-9bc5-6bdc9f6fd54d","issuer":"ray-ca-issuer-counter-auth-hw6hh","error":"issuers.cert-manager.io \"ray-ca-issuer-counter-auth-hw6hh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:247\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:22.807Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"7adc30a5-f89d-4339-8d35-1c1513e7878b"}
{"level":"info","ts":"2026-09-18T10:11:22.836Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"7c4c2614-3656-43f3-911a-04492f769e22","namespacedName":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:22.836Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"7c4c2614-3656-43f3-911a-04492f769e22"}
{"level":"info","ts":"2026-09-18T10:11:22.836Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"7c4c2614-3656-43f3-911a-04492f769e22","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:11:22.836Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"7c4c2614-3656-43f3-911a-04492f769e22","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:11:22.846Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"760d24f2-b0ce-45cb-8f51-0b857bdc3040"}
{"level":"error","ts":"2026-09-18T10:11:22.846Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"d97c3680-1818-4628-9bc5-6bdc9f6fd54d","certificate":"ray-ca-certificate-counter-auth-hw6hh","error":"certificates.cert-manager.io \"ray-ca-certificate-counter-auth-hw6hh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:22.895Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"d97c3680-1818-4628-9bc5-6bdc9f6fd54d","errorCount":2}
{"level":"info","ts":"2026-09-18T10:11:22.895Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"d97c3680-1818-4628-9bc5-6bdc9f6fd54d"}
{"level":"error","ts":"2026-09-18T10:11:22.895Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"d97c3680-1818-4628-9bc5-6bdc9f6fd54d","error":"issuers.cert-manager.io \"ray-ca-issuer-counter-auth-hw6hh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:22.900Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"ee7c88a6-355d-4105-a036-6956f5c1f7f4","namespace":"test-ns-ltx56","clusterName":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:22.900Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"counter-auth-hw6hh","reconcileID":"ee7c88a6-355d-4105-a036-6956f5c1f7f4","namespace":"test-ns-ltx56","clusterName":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:23.104Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"54254b85-7d40-466f-b897-11c0aa8cd2be","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:23.104Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"54254b85-7d40-466f-b897-11c0aa8cd2be","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T10:11:23.130Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"54254b85-7d40-466f-b897-11c0aa8cd2be"}
{"level":"info","ts":"2026-09-18T10:11:23.130Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"54254b85-7d40-466f-b897-11c0aa8cd2be","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T10:11:23.130Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"54254b85-7d40-466f-b897-11c0aa8cd2be","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:11:23.130Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"54254b85-7d40-466f-b897-11c0aa8cd2be","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:11:23.144Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"3cb182ea-8964-42e2-a859-190806f99de4","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:23.144Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"3cb182ea-8964-42e2-a859-190806f99de4"}
{"level":"info","ts":"2026-09-18T10:11:23.144Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"3cb182ea-8964-42e2-a859-190806f99de4","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T10:11:23.144Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"3cb182ea-8964-42e2-a859-190806f99de4","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-8cwf7","rayClusterName":"ttl-initializing-gcjcs","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:11:23.144Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"3cb182ea-8964-42e2-a859-190806f99de4","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:11:23.153Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"3cb182ea-8964-42e2-a859-190806f99de4","error":"Operation cannot be fulfilled on rayjobs.ray.io \"ttl-initializing\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T10:11:23.153Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"3cb182ea-8964-42e2-a859-190806f99de4"}
{"level":"error","ts":"2026-09-18T10:11:23.153Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"3cb182ea-8964-42e2-a859-190806f99de4","error":"Operation cannot be fulfilled on rayjobs.ray.io \"ttl-initializing\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.153Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"abe1693d-a3d5-4c76-9685-3caa9c0e1c53","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:23.153Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"abe1693d-a3d5-4c76-9685-3caa9c0e1c53","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.153Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"abe1693d-a3d5-4c76-9685-3caa9c0e1c53","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:23.153Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"abe1693d-a3d5-4c76-9685-3caa9c0e1c53","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:23.158Z","logger":"raycluster-resource","msg":"default","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.158Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"772878f5-7308-441e-9b96-676efc70d10c","rayCluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"772878f5-7308-441e-9b96-676efc70d10c","rayCluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"abe1693d-a3d5-4c76-9685-3caa9c0e1c53","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"abe1693d-a3d5-4c76-9685-3caa9c0e1c53"}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"abe1693d-a3d5-4c76-9685-3caa9c0e1c53","RayCluster":"ttl-initializing-rmrfw","State":""}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"abe1693d-a3d5-4c76-9685-3caa9c0e1c53","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"45f7dc3e-493e-4608-bfd1-bd75b11be276","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"ffefa2bc-4c99-4f04-b3e1-1855dba01f21","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"ffefa2bc-4c99-4f04-b3e1-1855dba01f21","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"ffefa2bc-4c99-4f04-b3e1-1855dba01f21","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"ffefa2bc-4c99-4f04-b3e1-1855dba01f21"}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"ffefa2bc-4c99-4f04-b3e1-1855dba01f21","RayCluster":"ttl-initializing-rmrfw","State":""}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"ffefa2bc-4c99-4f04-b3e1-1855dba01f21","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"f2413a64-2020-4663-b865-98d736c2c478","namespacedName":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"f2413a64-2020-4663-b865-98d736c2c478","mode":"IntegratedOAuth","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"f2413a64-2020-4663-b865-98d736c2c478","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"f2413a64-2020-4663-b865-98d736c2c478","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"195298d6-5088-41d9-9f3b-3a1ea87becb9"}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"195298d6-5088-41d9-9f3b-3a1ea87becb9"}
{"level":"info","ts":"2026-09-18T10:11:23.172Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"195298d6-5088-41d9-9f3b-3a1ea87becb9"}
{"level":"info","ts":"2026-09-18T10:11:23.173Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"aece32f3-1161-40ad-bf82-f8de47e6ff70","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:23.173Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"aece32f3-1161-40ad-bf82-f8de47e6ff70","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.173Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"aece32f3-1161-40ad-bf82-f8de47e6ff70","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:23.173Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"aece32f3-1161-40ad-bf82-f8de47e6ff70"}
{"level":"info","ts":"2026-09-18T10:11:23.173Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"aece32f3-1161-40ad-bf82-f8de47e6ff70","RayCluster":"ttl-initializing-rmrfw","State":""}
{"level":"info","ts":"2026-09-18T10:11:23.173Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"aece32f3-1161-40ad-bf82-f8de47e6ff70","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:11:23.179Z","logger":"raycluster-resource","msg":"default","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.179Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:11:23.188Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"195298d6-5088-41d9-9f3b-3a1ea87becb9","name":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:23.188Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"195298d6-5088-41d9-9f3b-3a1ea87becb9","error":"head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"error","ts":"2026-09-18T10:11:23.188Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"195298d6-5088-41d9-9f3b-3a1ea87becb9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.188Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"195298d6-5088-41d9-9f3b-3a1ea87becb9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"info","ts":"2026-09-18T10:11:23.188Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"195298d6-5088-41d9-9f3b-3a1ea87becb9","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.188Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"195298d6-5088-41d9-9f3b-3a1ea87becb9","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.188Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"195298d6-5088-41d9-9f3b-3a1ea87becb9","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.188Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"195298d6-5088-41d9-9f3b-3a1ea87becb9","name":"ttl-initializing-rmrfw","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:11:23.189Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"3f85ccde-0db7-4d23-8cbb-1a576dcb03f6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:23.189Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"3f85ccde-0db7-4d23-8cbb-1a576dcb03f6","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.196Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"45f7dc3e-493e-4608-bfd1-bd75b11be276","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"3f85ccde-0db7-4d23-8cbb-1a576dcb03f6","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"ba9796b6-897e-4d18-ac6b-41174a8c916a","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"3f85ccde-0db7-4d23-8cbb-1a576dcb03f6"}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"3f85ccde-0db7-4d23-8cbb-1a576dcb03f6","RayCluster":"ttl-initializing-rmrfw","State":""}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"3f85ccde-0db7-4d23-8cbb-1a576dcb03f6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:11:23.202Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"195298d6-5088-41d9-9f3b-3a1ea87becb9","name":"ttl-initializing-rmrfw","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ttl-initializing-rmrfw\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw"}}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"195298d6-5088-41d9-9f3b-3a1ea87becb9"}
{"level":"info","ts":"2026-09-18T10:11:23.198Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"f2413a64-2020-4663-b865-98d736c2c478","name":"ttl-initializing-rmrfw-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"error","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"195298d6-5088-41d9-9f3b-3a1ea87becb9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ac401fce-9806-4e09-9be1-226cdede8887"}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ac401fce-9806-4e09-9be1-226cdede8887"}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ac401fce-9806-4e09-9be1-226cdede8887"}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ac401fce-9806-4e09-9be1-226cdede8887","1 head service found":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ac401fce-9806-4e09-9be1-226cdede8887","error":"head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"error","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ac401fce-9806-4e09-9be1-226cdede8887","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ac401fce-9806-4e09-9be1-226cdede8887","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"info","ts":"2026-09-18T10:11:23.209Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"772878f5-7308-441e-9b96-676efc70d10c","rayCluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"772878f5-7308-441e-9b96-676efc70d10c","rayCluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ac401fce-9806-4e09-9be1-226cdede8887","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ac401fce-9806-4e09-9be1-226cdede8887","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ac401fce-9806-4e09-9be1-226cdede8887","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.266Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ac401fce-9806-4e09-9be1-226cdede8887","name":"ttl-initializing-rmrfw","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:11:23.277Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"f2413a64-2020-4663-b865-98d736c2c478","name":"test-ns-ltx56-ttl-initializing-rmrfw","operation":"created"}
{"level":"info","ts":"2026-09-18T10:11:23.280Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"4835dbd8-eaf7-4065-b8ef-bd35d2c6eaab","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:23.280Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"4835dbd8-eaf7-4065-b8ef-bd35d2c6eaab","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.280Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"4835dbd8-eaf7-4065-b8ef-bd35d2c6eaab","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:23.280Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"4835dbd8-eaf7-4065-b8ef-bd35d2c6eaab"}
{"level":"info","ts":"2026-09-18T10:11:23.280Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"4835dbd8-eaf7-4065-b8ef-bd35d2c6eaab","RayCluster":"ttl-initializing-rmrfw","State":""}
{"level":"info","ts":"2026-09-18T10:11:23.280Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"4835dbd8-eaf7-4065-b8ef-bd35d2c6eaab","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:23.280Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"4835dbd8-eaf7-4065-b8ef-bd35d2c6eaab","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:11:23.282Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ac401fce-9806-4e09-9be1-226cdede8887"}
{"level":"error","ts":"2026-09-18T10:11:23.282Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ac401fce-9806-4e09-9be1-226cdede8887","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.282Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"772878f5-7308-441e-9b96-676efc70d10c","rayCluster":"ttl-initializing-rmrfw","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T10:11:23.282Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"772878f5-7308-441e-9b96-676efc70d10c","rayCluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.282Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9231fd14-2f51-4fe0-9d38-3bd48311e805"}
{"level":"info","ts":"2026-09-18T10:11:23.282Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9231fd14-2f51-4fe0-9d38-3bd48311e805"}
{"level":"info","ts":"2026-09-18T10:11:23.282Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9231fd14-2f51-4fe0-9d38-3bd48311e805"}
{"level":"info","ts":"2026-09-18T10:11:23.282Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9231fd14-2f51-4fe0-9d38-3bd48311e805","1 head service found":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:23.283Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9231fd14-2f51-4fe0-9d38-3bd48311e805","error":"head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"error","ts":"2026-09-18T10:11:23.283Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9231fd14-2f51-4fe0-9d38-3bd48311e805","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.283Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9231fd14-2f51-4fe0-9d38-3bd48311e805","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"info","ts":"2026-09-18T10:11:23.283Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9231fd14-2f51-4fe0-9d38-3bd48311e805","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.283Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9231fd14-2f51-4fe0-9d38-3bd48311e805","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.283Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9231fd14-2f51-4fe0-9d38-3bd48311e805","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.283Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9231fd14-2f51-4fe0-9d38-3bd48311e805"}
{"level":"error","ts":"2026-09-18T10:11:23.283Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9231fd14-2f51-4fe0-9d38-3bd48311e805","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.289Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"f2413a64-2020-4663-b865-98d736c2c478","name":"kube-rbac-proxy-config-ttl-initializing-rmrfw","operation":"created"}
{"level":"info","ts":"2026-09-18T10:11:23.289Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"ba9796b6-897e-4d18-ac6b-41174a8c916a","name":"ttl-initializing-rmrfw-head"}
{"level":"info","ts":"2026-09-18T10:11:23.292Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"462e28a9-46b3-4552-a8f3-4519fc2ea2d0"}
{"level":"info","ts":"2026-09-18T10:11:23.292Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"462e28a9-46b3-4552-a8f3-4519fc2ea2d0"}
{"level":"info","ts":"2026-09-18T10:11:23.292Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"462e28a9-46b3-4552-a8f3-4519fc2ea2d0"}
{"level":"info","ts":"2026-09-18T10:11:23.292Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"462e28a9-46b3-4552-a8f3-4519fc2ea2d0","1 head service found":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:23.293Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"462e28a9-46b3-4552-a8f3-4519fc2ea2d0","error":"head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"error","ts":"2026-09-18T10:11:23.293Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"462e28a9-46b3-4552-a8f3-4519fc2ea2d0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.293Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"462e28a9-46b3-4552-a8f3-4519fc2ea2d0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"info","ts":"2026-09-18T10:11:23.293Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"462e28a9-46b3-4552-a8f3-4519fc2ea2d0","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.293Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"462e28a9-46b3-4552-a8f3-4519fc2ea2d0","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.293Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"462e28a9-46b3-4552-a8f3-4519fc2ea2d0","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.293Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"462e28a9-46b3-4552-a8f3-4519fc2ea2d0"}
{"level":"error","ts":"2026-09-18T10:11:23.293Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"462e28a9-46b3-4552-a8f3-4519fc2ea2d0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:11:23.305Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"f2413a64-2020-4663-b865-98d736c2c478","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ttl-initializing-rmrfw\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.305Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"f2413a64-2020-4663-b865-98d736c2c478"}
{"level":"error","ts":"2026-09-18T10:11:23.306Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"f2413a64-2020-4663-b865-98d736c2c478","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ttl-initializing-rmrfw\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.306Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"29cf8a9b-cfff-43bc-9e7a-b96a64e897c0","namespacedName":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:23.306Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"29cf8a9b-cfff-43bc-9e7a-b96a64e897c0","mode":"IntegratedOAuth","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.306Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"29cf8a9b-cfff-43bc-9e7a-b96a64e897c0","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.366Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8830cba3-2353-48a0-b5b4-7735b9564604","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:23.366Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8830cba3-2353-48a0-b5b4-7735b9564604","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.366Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8830cba3-2353-48a0-b5b4-7735b9564604","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:23.366Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8830cba3-2353-48a0-b5b4-7735b9564604"}
{"level":"info","ts":"2026-09-18T10:11:23.366Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8830cba3-2353-48a0-b5b4-7735b9564604","RayCluster":"ttl-initializing-rmrfw","State":""}
{"level":"info","ts":"2026-09-18T10:11:23.366Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8830cba3-2353-48a0-b5b4-7735b9564604","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:23.366Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"2de9fa5a-b6ff-4c72-a29b-2360e6b2dc68"}
{"level":"info","ts":"2026-09-18T10:11:23.366Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"2de9fa5a-b6ff-4c72-a29b-2360e6b2dc68"}
{"level":"info","ts":"2026-09-18T10:11:23.366Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"2de9fa5a-b6ff-4c72-a29b-2360e6b2dc68"}
{"level":"info","ts":"2026-09-18T10:11:23.366Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"2de9fa5a-b6ff-4c72-a29b-2360e6b2dc68","1 head service found":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:23.366Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"772878f5-7308-441e-9b96-676efc70d10c","rayCluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.367Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"2de9fa5a-b6ff-4c72-a29b-2360e6b2dc68","error":"head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"error","ts":"2026-09-18T10:11:23.367Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"2de9fa5a-b6ff-4c72-a29b-2360e6b2dc68","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.367Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"2de9fa5a-b6ff-4c72-a29b-2360e6b2dc68","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"info","ts":"2026-09-18T10:11:23.367Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"2de9fa5a-b6ff-4c72-a29b-2360e6b2dc68","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.367Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"2de9fa5a-b6ff-4c72-a29b-2360e6b2dc68","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.367Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"2de9fa5a-b6ff-4c72-a29b-2360e6b2dc68","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.367Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"2de9fa5a-b6ff-4c72-a29b-2360e6b2dc68"}
{"level":"error","ts":"2026-09-18T10:11:23.367Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"2de9fa5a-b6ff-4c72-a29b-2360e6b2dc68","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.378Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"ba9796b6-897e-4d18-ac6b-41174a8c916a","name":"ttl-initializing-rmrfw-workers"}
{"level":"info","ts":"2026-09-18T10:11:23.378Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"ba9796b6-897e-4d18-ac6b-41174a8c916a","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.378Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"a4b924c3-c3b7-486a-9648-a88d37b89f1a","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.379Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"29cf8a9b-cfff-43bc-9e7a-b96a64e897c0","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.379Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"585e3ec9-3730-45b0-a24f-53d10f6ca2a4","namespacedName":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:23.379Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"585e3ec9-3730-45b0-a24f-53d10f6ca2a4","mode":"IntegratedOAuth","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.379Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"585e3ec9-3730-45b0-a24f-53d10f6ca2a4","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"f901c415-5565-492f-8ab6-31483273e998"}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"f901c415-5565-492f-8ab6-31483273e998"}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"f901c415-5565-492f-8ab6-31483273e998"}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"f901c415-5565-492f-8ab6-31483273e998","1 head service found":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"f901c415-5565-492f-8ab6-31483273e998","error":"head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"error","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"f901c415-5565-492f-8ab6-31483273e998","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"f901c415-5565-492f-8ab6-31483273e998","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"f901c415-5565-492f-8ab6-31483273e998","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"f901c415-5565-492f-8ab6-31483273e998","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"f901c415-5565-492f-8ab6-31483273e998","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"f901c415-5565-492f-8ab6-31483273e998"}
{"level":"error","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"f901c415-5565-492f-8ab6-31483273e998","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8e53f9bf-73e2-4782-8772-4c0ef9927f93","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8e53f9bf-73e2-4782-8772-4c0ef9927f93","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8e53f9bf-73e2-4782-8772-4c0ef9927f93","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"772878f5-7308-441e-9b96-676efc70d10c","rayCluster":"ttl-initializing-rmrfw","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8e53f9bf-73e2-4782-8772-4c0ef9927f93"}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8e53f9bf-73e2-4782-8772-4c0ef9927f93","RayCluster":"ttl-initializing-rmrfw","State":""}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8e53f9bf-73e2-4782-8772-4c0ef9927f93","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:23.466Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8e53f9bf-73e2-4782-8772-4c0ef9927f93","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:11:23.467Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"585e3ec9-3730-45b0-a24f-53d10f6ca2a4","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.475Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"a4b924c3-c3b7-486a-9648-a88d37b89f1a","name":"ttl-initializing-rmrfw-head"}
{"level":"info","ts":"2026-09-18T10:11:23.480Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"772878f5-7308-441e-9b96-676efc70d10c","rayCluster":"ttl-initializing-rmrfw","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:11:23.480Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"772878f5-7308-441e-9b96-676efc70d10c"}
{"level":"info","ts":"2026-09-18T10:11:23.480Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"3db18c62-ea57-42d7-af86-340d6e52d795","rayCluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.480Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"3db18c62-ea57-42d7-af86-340d6e52d795"}
{"level":"info","ts":"2026-09-18T10:11:23.485Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"05ff3531-f47c-4436-9160-f52cd68d7776","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:23.485Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"05ff3531-f47c-4436-9160-f52cd68d7776","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.485Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"05ff3531-f47c-4436-9160-f52cd68d7776","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:23.485Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"05ff3531-f47c-4436-9160-f52cd68d7776"}
{"level":"info","ts":"2026-09-18T10:11:23.485Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"05ff3531-f47c-4436-9160-f52cd68d7776","RayCluster":"ttl-initializing-rmrfw","State":""}
{"level":"info","ts":"2026-09-18T10:11:23.485Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"05ff3531-f47c-4436-9160-f52cd68d7776","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:23.489Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"a4b924c3-c3b7-486a-9648-a88d37b89f1a","name":"ttl-initializing-rmrfw-workers"}
{"level":"info","ts":"2026-09-18T10:11:23.489Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"a4b924c3-c3b7-486a-9648-a88d37b89f1a","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.489Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"13855631-b612-472e-ae4a-38d80d7810ac","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.504Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"13855631-b612-472e-ae4a-38d80d7810ac","name":"ttl-initializing-rmrfw-head"}
{"level":"info","ts":"2026-09-18T10:11:23.526Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"13855631-b612-472e-ae4a-38d80d7810ac","name":"ttl-initializing-rmrfw-workers"}
{"level":"info","ts":"2026-09-18T10:11:23.526Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"13855631-b612-472e-ae4a-38d80d7810ac","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:23.615Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"06f080df-635b-4ef3-80a8-66e6bc5cbd8d","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:23.615Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"06f080df-635b-4ef3-80a8-66e6bc5cbd8d","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:11:22 +0000 UTC","Now":"2026-09-18T10:11:23.615Z","ShutdownTime":"2026-09-18T10:11:22.000Z"}
{"level":"info","ts":"2026-09-18T10:11:23.615Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"06f080df-635b-4ef3-80a8-66e6bc5cbd8d","RayCluster":{"name":"counter-auth-hw6hh","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:23.615Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"06f080df-635b-4ef3-80a8-66e6bc5cbd8d","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T10:11:23.615Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"06f080df-635b-4ef3-80a8-66e6bc5cbd8d","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-hw6hh"}
{"level":"info","ts":"2026-09-18T10:11:23.627Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"988b7eab-0a74-4a3f-8e3e-8c2b37b8f8a0"}
{"level":"info","ts":"2026-09-18T10:11:23.627Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"988b7eab-0a74-4a3f-8e3e-8c2b37b8f8a0"}
{"level":"info","ts":"2026-09-18T10:11:23.627Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"988b7eab-0a74-4a3f-8e3e-8c2b37b8f8a0"}
{"level":"info","ts":"2026-09-18T10:11:23.627Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"988b7eab-0a74-4a3f-8e3e-8c2b37b8f8a0","1 head service found":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:23.627Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"988b7eab-0a74-4a3f-8e3e-8c2b37b8f8a0","error":"head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"error","ts":"2026-09-18T10:11:23.627Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"988b7eab-0a74-4a3f-8e3e-8c2b37b8f8a0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.627Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"988b7eab-0a74-4a3f-8e3e-8c2b37b8f8a0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"info","ts":"2026-09-18T10:11:23.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"988b7eab-0a74-4a3f-8e3e-8c2b37b8f8a0","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"988b7eab-0a74-4a3f-8e3e-8c2b37b8f8a0","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"988b7eab-0a74-4a3f-8e3e-8c2b37b8f8a0","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.627Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"988b7eab-0a74-4a3f-8e3e-8c2b37b8f8a0"}
{"level":"error","ts":"2026-09-18T10:11:23.627Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"988b7eab-0a74-4a3f-8e3e-8c2b37b8f8a0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.949Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"93fc3c90-49ec-4991-a93d-581ddd701f92"}
{"level":"info","ts":"2026-09-18T10:11:23.949Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"93fc3c90-49ec-4991-a93d-581ddd701f92"}
{"level":"info","ts":"2026-09-18T10:11:23.949Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"93fc3c90-49ec-4991-a93d-581ddd701f92"}
{"level":"info","ts":"2026-09-18T10:11:23.949Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"93fc3c90-49ec-4991-a93d-581ddd701f92","1 head service found":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:23.949Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"93fc3c90-49ec-4991-a93d-581ddd701f92","error":"head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"error","ts":"2026-09-18T10:11:23.949Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"93fc3c90-49ec-4991-a93d-581ddd701f92","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:23.949Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"93fc3c90-49ec-4991-a93d-581ddd701f92","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"info","ts":"2026-09-18T10:11:23.949Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"93fc3c90-49ec-4991-a93d-581ddd701f92","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.949Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"93fc3c90-49ec-4991-a93d-581ddd701f92","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.949Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"93fc3c90-49ec-4991-a93d-581ddd701f92","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:23.949Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"93fc3c90-49ec-4991-a93d-581ddd701f92"}
{"level":"error","ts":"2026-09-18T10:11:23.949Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"93fc3c90-49ec-4991-a93d-581ddd701f92","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:24.589Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9f352b2b-8684-4d99-bb1c-0ad3f039f860"}
{"level":"info","ts":"2026-09-18T10:11:24.589Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9f352b2b-8684-4d99-bb1c-0ad3f039f860"}
{"level":"info","ts":"2026-09-18T10:11:24.589Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9f352b2b-8684-4d99-bb1c-0ad3f039f860"}
{"level":"info","ts":"2026-09-18T10:11:24.589Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9f352b2b-8684-4d99-bb1c-0ad3f039f860","1 head service found":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:24.589Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9f352b2b-8684-4d99-bb1c-0ad3f039f860","error":"head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"error","ts":"2026-09-18T10:11:24.589Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9f352b2b-8684-4d99-bb1c-0ad3f039f860","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:24.589Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9f352b2b-8684-4d99-bb1c-0ad3f039f860","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found"}
{"level":"info","ts":"2026-09-18T10:11:24.590Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9f352b2b-8684-4d99-bb1c-0ad3f039f860","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:24.590Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9f352b2b-8684-4d99-bb1c-0ad3f039f860","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:24.590Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9f352b2b-8684-4d99-bb1c-0ad3f039f860","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-initializing-rmrfw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:24.590Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9f352b2b-8684-4d99-bb1c-0ad3f039f860"}
{"level":"error","ts":"2026-09-18T10:11:24.590Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"9f352b2b-8684-4d99-bb1c-0ad3f039f860","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-initializing-rmrfw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:25.871Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0"}
{"level":"info","ts":"2026-09-18T10:11:25.871Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0"}
{"level":"info","ts":"2026-09-18T10:11:25.871Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0"}
{"level":"info","ts":"2026-09-18T10:11:25.871Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","1 head service found":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:25.871Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0"}
{"level":"info","ts":"2026-09-18T10:11:25.871Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:11:25.871Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0"}
{"level":"info","ts":"2026-09-18T10:11:25.871Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","mode":"IntegratedOAuth","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:25.871Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","cluster":"ttl-initializing-rmrfw","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:11:25.871Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","cluster":"ttl-initializing-rmrfw","authType":"IntegratedOAuth","serviceAccount":"ttl-initializing-rmrfw-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T10:11:25.871Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ttl-initializing-rmrfw","ray.io/group":"headgroup","ray.io/identifier":"ttl-initializing-rmrfw-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:11:25.871Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T10:11:25.871Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:11:25.871Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:11:25.871Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","enabled":true}
{"level":"info","ts":"2026-09-18T10:11:25.892Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","name":"ttl-initializing-rmrfw-head-mnhjd"}
{"level":"info","ts":"2026-09-18T10:11:25.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:25.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:25.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:11:25.892Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:11:25.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:25.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:11:25.892Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0"}
{"level":"info","ts":"2026-09-18T10:11:25.892Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","nodeType":"worker","rayStartParams":{"address":"ttl-initializing-rmrfw-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:11:25.892Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","rayStartCmd":"ray start  --address=ttl-initializing-rmrfw-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:11:25.892Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=ttl-initializing-rmrfw-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:11:25.892Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","enabled":true}
{"level":"info","ts":"2026-09-18T10:11:25.912Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","name":"ttl-initializing-rmrfw-small-group-worker-88qtq"}
{"level":"info","ts":"2026-09-18T10:11:25.912Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","podName":"ttl-initializing-rmrfw-small-group-worker-88qtq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:25.912Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","podName":"ttl-initializing-rmrfw-small-group-worker-88qtq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:25.913Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"61499b54-7fff-410d-91a4-a479995bc3d0","name":"ttl-initializing-rmrfw","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:11:25.937Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"a5d79c73-05e7-4c82-a725-f3b3ad29a7aa","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:25.937Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"eaf0f1b7-d744-420b-98d6-d77203b2e92f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:25.937Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"eaf0f1b7-d744-420b-98d6-d77203b2e92f","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:25.937Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"eaf0f1b7-d744-420b-98d6-d77203b2e92f","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:25.937Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"eaf0f1b7-d744-420b-98d6-d77203b2e92f"}
{"level":"info","ts":"2026-09-18T10:11:25.937Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"eaf0f1b7-d744-420b-98d6-d77203b2e92f","RayCluster":"ttl-initializing-rmrfw","State":""}
{"level":"info","ts":"2026-09-18T10:11:25.937Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"eaf0f1b7-d744-420b-98d6-d77203b2e92f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:25.937Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"eaf0f1b7-d744-420b-98d6-d77203b2e92f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:11:25.938Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"8068501a-02d3-45b1-9b88-886c33ee16c4","rayCluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:25.938Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"8068501a-02d3-45b1-9b88-886c33ee16c4","rayCluster":"ttl-initializing-rmrfw","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:11:26.066Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"fa94360b-4db1-4037-aae8-7d71d746b50d"}
{"level":"info","ts":"2026-09-18T10:11:26.066Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"fa94360b-4db1-4037-aae8-7d71d746b50d"}
{"level":"info","ts":"2026-09-18T10:11:26.066Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"fa94360b-4db1-4037-aae8-7d71d746b50d"}
{"level":"info","ts":"2026-09-18T10:11:26.066Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"fa94360b-4db1-4037-aae8-7d71d746b50d","1 head service found":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:26.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"fa94360b-4db1-4037-aae8-7d71d746b50d","Found 1 head Pod":"ttl-initializing-rmrfw-head-mnhjd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:26.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"fa94360b-4db1-4037-aae8-7d71d746b50d","head Pod":"ttl-initializing-rmrfw-head-mnhjd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-initializing-rmrfw-head-mnhjd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:26.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"fa94360b-4db1-4037-aae8-7d71d746b50d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:26.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"fa94360b-4db1-4037-aae8-7d71d746b50d","worker Pod":"ttl-initializing-rmrfw-small-group-worker-88qtq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-initializing-rmrfw-small-group-worker-88qtq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:26.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"fa94360b-4db1-4037-aae8-7d71d746b50d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:26.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"fa94360b-4db1-4037-aae8-7d71d746b50d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:26.067Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"fa94360b-4db1-4037-aae8-7d71d746b50d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:26.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"fa94360b-4db1-4037-aae8-7d71d746b50d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:26.067Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"fa94360b-4db1-4037-aae8-7d71d746b50d","podName":"ttl-initializing-rmrfw-head-mnhjd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:26.067Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"fa94360b-4db1-4037-aae8-7d71d746b50d","podName":"ttl-initializing-rmrfw-head-mnhjd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:26.067Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"fa94360b-4db1-4037-aae8-7d71d746b50d","name":"ttl-initializing-rmrfw","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:11:26.068Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"d6b63cbd-694b-4ef1-8ce8-e0ccc7ef2127","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:26.068Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"d6b63cbd-694b-4ef1-8ce8-e0ccc7ef2127","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:26.068Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"d6b63cbd-694b-4ef1-8ce8-e0ccc7ef2127","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:26.068Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"d6b63cbd-694b-4ef1-8ce8-e0ccc7ef2127"}
{"level":"info","ts":"2026-09-18T10:11:26.068Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"d6b63cbd-694b-4ef1-8ce8-e0ccc7ef2127","RayCluster":"ttl-initializing-rmrfw","State":""}
{"level":"info","ts":"2026-09-18T10:11:26.068Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"d6b63cbd-694b-4ef1-8ce8-e0ccc7ef2127","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:26.166Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"a5d79c73-05e7-4c82-a725-f3b3ad29a7aa","name":"ttl-initializing-rmrfw-head"}
{"level":"info","ts":"2026-09-18T10:11:26.166Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"27b3fb8d-e5c7-463a-aae6-fd0f9233f725","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:26.166Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"27b3fb8d-e5c7-463a-aae6-fd0f9233f725","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:26.166Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"27b3fb8d-e5c7-463a-aae6-fd0f9233f725","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:26.166Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ed89ab7e-3069-450d-a1bd-8fa88b5d0580"}
{"level":"info","ts":"2026-09-18T10:11:26.166Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ed89ab7e-3069-450d-a1bd-8fa88b5d0580"}
{"level":"info","ts":"2026-09-18T10:11:26.166Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ed89ab7e-3069-450d-a1bd-8fa88b5d0580"}
{"level":"info","ts":"2026-09-18T10:11:26.166Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ed89ab7e-3069-450d-a1bd-8fa88b5d0580","1 head service found":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:26.166Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"27b3fb8d-e5c7-463a-aae6-fd0f9233f725"}
{"level":"info","ts":"2026-09-18T10:11:26.166Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"27b3fb8d-e5c7-463a-aae6-fd0f9233f725","RayCluster":"ttl-initializing-rmrfw","State":""}
{"level":"info","ts":"2026-09-18T10:11:26.166Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"27b3fb8d-e5c7-463a-aae6-fd0f9233f725","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:26.166Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"27b3fb8d-e5c7-463a-aae6-fd0f9233f725","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:11:26.167Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ed89ab7e-3069-450d-a1bd-8fa88b5d0580","Found 1 head Pod":"ttl-initializing-rmrfw-head-mnhjd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:26.167Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ed89ab7e-3069-450d-a1bd-8fa88b5d0580","head Pod":"ttl-initializing-rmrfw-head-mnhjd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-initializing-rmrfw-head-mnhjd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:26.167Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ed89ab7e-3069-450d-a1bd-8fa88b5d0580","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:26.167Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ed89ab7e-3069-450d-a1bd-8fa88b5d0580","worker Pod":"ttl-initializing-rmrfw-small-group-worker-88qtq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-initializing-rmrfw-small-group-worker-88qtq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:26.167Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ed89ab7e-3069-450d-a1bd-8fa88b5d0580","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:26.167Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ed89ab7e-3069-450d-a1bd-8fa88b5d0580","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:26.167Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ed89ab7e-3069-450d-a1bd-8fa88b5d0580","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:26.167Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ed89ab7e-3069-450d-a1bd-8fa88b5d0580","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:26.167Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ed89ab7e-3069-450d-a1bd-8fa88b5d0580","podName":"ttl-initializing-rmrfw-head-mnhjd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:26.167Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ed89ab7e-3069-450d-a1bd-8fa88b5d0580","podName":"ttl-initializing-rmrfw-head-mnhjd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:26.167Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"2bdf1fa1-3f2b-40f8-8647-ad6dfa7c67cb","rayCluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:26.167Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ed89ab7e-3069-450d-a1bd-8fa88b5d0580","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:11:26.167Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"ed89ab7e-3069-450d-a1bd-8fa88b5d0580","seconds":300}
{"level":"info","ts":"2026-09-18T10:11:26.167Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"2bdf1fa1-3f2b-40f8-8647-ad6dfa7c67cb","rayCluster":"ttl-initializing-rmrfw","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:11:26.181Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8783c7dc-95f5-4b83-a225-fbb309da7c29","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:26.181Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8783c7dc-95f5-4b83-a225-fbb309da7c29","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:26.181Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8783c7dc-95f5-4b83-a225-fbb309da7c29","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:26.181Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8783c7dc-95f5-4b83-a225-fbb309da7c29"}
{"level":"info","ts":"2026-09-18T10:11:26.181Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8783c7dc-95f5-4b83-a225-fbb309da7c29","RayCluster":"ttl-initializing-rmrfw","State":""}
{"level":"info","ts":"2026-09-18T10:11:26.181Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8783c7dc-95f5-4b83-a225-fbb309da7c29","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:26.181Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8783c7dc-95f5-4b83-a225-fbb309da7c29","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:11:26.187Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"a5d79c73-05e7-4c82-a725-f3b3ad29a7aa","name":"ttl-initializing-rmrfw-workers"}
{"level":"info","ts":"2026-09-18T10:11:26.187Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"a5d79c73-05e7-4c82-a725-f3b3ad29a7aa","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:26.187Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"6546058a-5c3e-4231-b87c-0d459025b45a","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:26.193Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8783c7dc-95f5-4b83-a225-fbb309da7c29","error":"Operation cannot be fulfilled on rayjobs.ray.io \"ttl-initializing\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-09-18T10:11:26.193Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8783c7dc-95f5-4b83-a225-fbb309da7c29"}
{"level":"error","ts":"2026-09-18T10:11:26.193Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"8783c7dc-95f5-4b83-a225-fbb309da7c29","error":"Operation cannot be fulfilled on rayjobs.ray.io \"ttl-initializing\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:26.193Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"200e980d-24c7-4eeb-89e9-f21fa6592050","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:26.193Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"200e980d-24c7-4eeb-89e9-f21fa6592050","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:26.193Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"200e980d-24c7-4eeb-89e9-f21fa6592050","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:26.193Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"200e980d-24c7-4eeb-89e9-f21fa6592050"}
{"level":"info","ts":"2026-09-18T10:11:26.193Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"200e980d-24c7-4eeb-89e9-f21fa6592050","RayCluster":"ttl-initializing-rmrfw","State":""}
{"level":"info","ts":"2026-09-18T10:11:26.193Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"200e980d-24c7-4eeb-89e9-f21fa6592050","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:26.198Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"beabae92-7c11-4186-a3cd-214ff684f87f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:26.199Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"beabae92-7c11-4186-a3cd-214ff684f87f","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:26.199Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"beabae92-7c11-4186-a3cd-214ff684f87f","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:26.199Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"beabae92-7c11-4186-a3cd-214ff684f87f"}
{"level":"info","ts":"2026-09-18T10:11:26.199Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"beabae92-7c11-4186-a3cd-214ff684f87f","RayCluster":"ttl-initializing-rmrfw","State":""}
{"level":"info","ts":"2026-09-18T10:11:26.199Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"beabae92-7c11-4186-a3cd-214ff684f87f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:26.204Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"6546058a-5c3e-4231-b87c-0d459025b45a","name":"ttl-initializing-rmrfw-head"}
{"level":"info","ts":"2026-09-18T10:11:26.216Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"6546058a-5c3e-4231-b87c-0d459025b45a","name":"ttl-initializing-rmrfw-workers"}
{"level":"info","ts":"2026-09-18T10:11:26.216Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"6546058a-5c3e-4231-b87c-0d459025b45a","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:26.498Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"8c24b063-3ab6-4445-ac10-b3d98b673b59"}
{"level":"info","ts":"2026-09-18T10:11:26.498Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"8c24b063-3ab6-4445-ac10-b3d98b673b59"}
{"level":"info","ts":"2026-09-18T10:11:26.498Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"8c24b063-3ab6-4445-ac10-b3d98b673b59"}
{"level":"info","ts":"2026-09-18T10:11:26.498Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"8c24b063-3ab6-4445-ac10-b3d98b673b59","1 head service found":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:26.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"8c24b063-3ab6-4445-ac10-b3d98b673b59","Found 1 head Pod":"ttl-initializing-rmrfw-head-mnhjd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:26.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"8c24b063-3ab6-4445-ac10-b3d98b673b59","head Pod":"ttl-initializing-rmrfw-head-mnhjd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-initializing-rmrfw-head-mnhjd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:26.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"8c24b063-3ab6-4445-ac10-b3d98b673b59","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:26.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"8c24b063-3ab6-4445-ac10-b3d98b673b59","worker Pod":"ttl-initializing-rmrfw-small-group-worker-88qtq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-initializing-rmrfw-small-group-worker-88qtq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:26.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"8c24b063-3ab6-4445-ac10-b3d98b673b59","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:26.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"8c24b063-3ab6-4445-ac10-b3d98b673b59","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:26.498Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"8c24b063-3ab6-4445-ac10-b3d98b673b59","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:26.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"8c24b063-3ab6-4445-ac10-b3d98b673b59","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:26.498Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"8c24b063-3ab6-4445-ac10-b3d98b673b59","podName":"ttl-initializing-rmrfw-head-mnhjd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:26.498Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"8c24b063-3ab6-4445-ac10-b3d98b673b59","podName":"ttl-initializing-rmrfw-head-mnhjd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:26.498Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"8c24b063-3ab6-4445-ac10-b3d98b673b59","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:11:26.498Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"8c24b063-3ab6-4445-ac10-b3d98b673b59","seconds":300}
{"level":"info","ts":"2026-09-18T10:11:26.529Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"acb01293-cfe1-4c95-bab4-ce3d752377b7"}
{"level":"info","ts":"2026-09-18T10:11:26.529Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"acb01293-cfe1-4c95-bab4-ce3d752377b7"}
{"level":"info","ts":"2026-09-18T10:11:26.529Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"acb01293-cfe1-4c95-bab4-ce3d752377b7"}
{"level":"info","ts":"2026-09-18T10:11:26.529Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"acb01293-cfe1-4c95-bab4-ce3d752377b7","1 head service found":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:26.529Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"acb01293-cfe1-4c95-bab4-ce3d752377b7","Found 1 head Pod":"ttl-initializing-rmrfw-head-mnhjd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:26.529Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"acb01293-cfe1-4c95-bab4-ce3d752377b7","head Pod":"ttl-initializing-rmrfw-head-mnhjd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-initializing-rmrfw-head-mnhjd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:26.529Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"acb01293-cfe1-4c95-bab4-ce3d752377b7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:26.529Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"acb01293-cfe1-4c95-bab4-ce3d752377b7","worker Pod":"ttl-initializing-rmrfw-small-group-worker-88qtq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-initializing-rmrfw-small-group-worker-88qtq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:26.529Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"acb01293-cfe1-4c95-bab4-ce3d752377b7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:26.529Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"acb01293-cfe1-4c95-bab4-ce3d752377b7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:26.529Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"acb01293-cfe1-4c95-bab4-ce3d752377b7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:26.529Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"acb01293-cfe1-4c95-bab4-ce3d752377b7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:26.529Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"acb01293-cfe1-4c95-bab4-ce3d752377b7","podName":"ttl-initializing-rmrfw-head-mnhjd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:26.529Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"acb01293-cfe1-4c95-bab4-ce3d752377b7","podName":"ttl-initializing-rmrfw-head-mnhjd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:26.530Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"acb01293-cfe1-4c95-bab4-ce3d752377b7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:11:26.530Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"acb01293-cfe1-4c95-bab4-ce3d752377b7","seconds":300}
{"level":"info","ts":"2026-09-18T10:11:26.755Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"e9247be7-179b-4139-bae7-a88a1d77c7c9"}
{"level":"info","ts":"2026-09-18T10:11:26.755Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"e9247be7-179b-4139-bae7-a88a1d77c7c9"}
{"level":"info","ts":"2026-09-18T10:11:26.755Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"e9247be7-179b-4139-bae7-a88a1d77c7c9"}
{"level":"info","ts":"2026-09-18T10:11:26.755Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"e9247be7-179b-4139-bae7-a88a1d77c7c9","1 head service found":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:26.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"e9247be7-179b-4139-bae7-a88a1d77c7c9","Found 1 head Pod":"ttl-initializing-rmrfw-head-mnhjd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:26.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"e9247be7-179b-4139-bae7-a88a1d77c7c9","head Pod":"ttl-initializing-rmrfw-head-mnhjd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-initializing-rmrfw-head-mnhjd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:26.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"e9247be7-179b-4139-bae7-a88a1d77c7c9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:26.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"e9247be7-179b-4139-bae7-a88a1d77c7c9","worker Pod":"ttl-initializing-rmrfw-small-group-worker-88qtq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-initializing-rmrfw-small-group-worker-88qtq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:26.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"e9247be7-179b-4139-bae7-a88a1d77c7c9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:26.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"e9247be7-179b-4139-bae7-a88a1d77c7c9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:26.755Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"e9247be7-179b-4139-bae7-a88a1d77c7c9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:26.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"e9247be7-179b-4139-bae7-a88a1d77c7c9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:26.755Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"e9247be7-179b-4139-bae7-a88a1d77c7c9","podName":"ttl-initializing-rmrfw-head-mnhjd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:26.755Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"e9247be7-179b-4139-bae7-a88a1d77c7c9","podName":"ttl-initializing-rmrfw-head-mnhjd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:26.756Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"e9247be7-179b-4139-bae7-a88a1d77c7c9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:11:26.756Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"e9247be7-179b-4139-bae7-a88a1d77c7c9","seconds":300}
{"level":"info","ts":"2026-09-18T10:11:27.638Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"bc6e945c-6f51-498e-b4f0-5b7b564755d0"}
{"level":"info","ts":"2026-09-18T10:11:27.638Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"bc6e945c-6f51-498e-b4f0-5b7b564755d0"}
{"level":"info","ts":"2026-09-18T10:11:27.638Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"bc6e945c-6f51-498e-b4f0-5b7b564755d0"}
{"level":"info","ts":"2026-09-18T10:11:27.638Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"bc6e945c-6f51-498e-b4f0-5b7b564755d0","1 head service found":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:27.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"bc6e945c-6f51-498e-b4f0-5b7b564755d0","Found 1 head Pod":"ttl-initializing-rmrfw-head-mnhjd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:27.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"bc6e945c-6f51-498e-b4f0-5b7b564755d0","head Pod":"ttl-initializing-rmrfw-head-mnhjd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-initializing-rmrfw-head-mnhjd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:27.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"bc6e945c-6f51-498e-b4f0-5b7b564755d0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:27.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"bc6e945c-6f51-498e-b4f0-5b7b564755d0","worker Pod":"ttl-initializing-rmrfw-small-group-worker-88qtq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-initializing-rmrfw-small-group-worker-88qtq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:27.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"bc6e945c-6f51-498e-b4f0-5b7b564755d0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:27.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"bc6e945c-6f51-498e-b4f0-5b7b564755d0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:27.638Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"bc6e945c-6f51-498e-b4f0-5b7b564755d0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:27.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"bc6e945c-6f51-498e-b4f0-5b7b564755d0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:27.639Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"bc6e945c-6f51-498e-b4f0-5b7b564755d0","podName":"ttl-initializing-rmrfw-head-mnhjd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:27.639Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"bc6e945c-6f51-498e-b4f0-5b7b564755d0","podName":"ttl-initializing-rmrfw-head-mnhjd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:27.639Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"bc6e945c-6f51-498e-b4f0-5b7b564755d0","name":"ttl-initializing-rmrfw","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.57","serviceIP":"10.132.0.57","podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:11:27.658Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"12c89ef8-663b-49b9-869b-e3460f9958e7","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:27.658Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"a7d64e0e-1de4-4c52-b112-2767acb114bc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:27.658Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"a7d64e0e-1de4-4c52-b112-2767acb114bc","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:27.658Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"a7d64e0e-1de4-4c52-b112-2767acb114bc","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:27.658Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"a7d64e0e-1de4-4c52-b112-2767acb114bc"}
{"level":"info","ts":"2026-09-18T10:11:27.658Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"a7d64e0e-1de4-4c52-b112-2767acb114bc","RayCluster":"ttl-initializing-rmrfw","State":""}
{"level":"info","ts":"2026-09-18T10:11:27.658Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"a7d64e0e-1de4-4c52-b112-2767acb114bc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.57","serviceIP":"10.132.0.57","podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:27.658Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"a7d64e0e-1de4-4c52-b112-2767acb114bc","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:11:27.658Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"9b984c8b-dce4-4078-97ba-c6415220c4ef","rayCluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:27.658Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"9b984c8b-dce4-4078-97ba-c6415220c4ef","pod":"ttl-initializing-rmrfw-head-mnhjd","ip":"10.132.0.57"}
{"level":"info","ts":"2026-09-18T10:11:27.658Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"9b984c8b-dce4-4078-97ba-c6415220c4ef","pod":"ttl-initializing-rmrfw-small-group-worker-88qtq","ip":"10.133.0.49"}
{"level":"info","ts":"2026-09-18T10:11:27.679Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"509e6986-9a81-478a-9413-f162e9839dc3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:27.679Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"509e6986-9a81-478a-9413-f162e9839dc3","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:27.679Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"509e6986-9a81-478a-9413-f162e9839dc3","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:27.679Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"509e6986-9a81-478a-9413-f162e9839dc3"}
{"level":"info","ts":"2026-09-18T10:11:27.679Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"509e6986-9a81-478a-9413-f162e9839dc3","RayCluster":"ttl-initializing-rmrfw","State":""}
{"level":"info","ts":"2026-09-18T10:11:27.679Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"509e6986-9a81-478a-9413-f162e9839dc3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.57","serviceIP":"10.132.0.57","podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.57","serviceIP":"10.132.0.57","podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:27.680Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"12c89ef8-663b-49b9-869b-e3460f9958e7","name":"ttl-initializing-rmrfw-head"}
{"level":"info","ts":"2026-09-18T10:11:27.680Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"9b984c8b-dce4-4078-97ba-c6415220c4ef","certificate":"ray-head-cert-ttl-initializing-rmrfw","dnsNames":["ttl-initializing-rmrfw-head-svc","localhost","ttl-initializing-rmrfw-head-svc.test-ns-ltx56.svc","ttl-initializing-rmrfw-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.57","10.133.0.49","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:11:27.695Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"12c89ef8-663b-49b9-869b-e3460f9958e7","name":"ttl-initializing-rmrfw-workers"}
{"level":"info","ts":"2026-09-18T10:11:27.695Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"12c89ef8-663b-49b9-869b-e3460f9958e7","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:27.697Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"9b984c8b-dce4-4078-97ba-c6415220c4ef","certificate":"ray-worker-cert-ttl-initializing-rmrfw","dnsNames":["ttl-initializing-rmrfw-worker-svc","localhost","ttl-initializing-rmrfw-small-group","ttl-initializing-rmrfw-worker-svc.test-ns-ltx56.svc","ttl-initializing-rmrfw-worker-svc.test-ns-ltx56.svc.cluster.local","*.ttl-initializing-rmrfw-worker-svc.test-ns-ltx56.svc","*.ttl-initializing-rmrfw-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.57","10.133.0.49","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:11:27.697Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"9b984c8b-dce4-4078-97ba-c6415220c4ef"}
{"level":"info","ts":"2026-09-18T10:11:28.067Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"239b0ae8-7aec-471d-8ff3-3248ebb35543"}
{"level":"info","ts":"2026-09-18T10:11:28.067Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"239b0ae8-7aec-471d-8ff3-3248ebb35543"}
{"level":"info","ts":"2026-09-18T10:11:28.067Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"239b0ae8-7aec-471d-8ff3-3248ebb35543"}
{"level":"info","ts":"2026-09-18T10:11:28.067Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"239b0ae8-7aec-471d-8ff3-3248ebb35543","1 head service found":"ttl-initializing-rmrfw-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:28.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"239b0ae8-7aec-471d-8ff3-3248ebb35543","Found 1 head Pod":"ttl-initializing-rmrfw-head-mnhjd","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:28.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"239b0ae8-7aec-471d-8ff3-3248ebb35543","head Pod":"ttl-initializing-rmrfw-head-mnhjd","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-initializing-rmrfw-head-mnhjd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:28.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"239b0ae8-7aec-471d-8ff3-3248ebb35543","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:28.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"239b0ae8-7aec-471d-8ff3-3248ebb35543","worker Pod":"ttl-initializing-rmrfw-small-group-worker-88qtq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-initializing-rmrfw-small-group-worker-88qtq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:28.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"239b0ae8-7aec-471d-8ff3-3248ebb35543","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:28.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"239b0ae8-7aec-471d-8ff3-3248ebb35543","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:28.067Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"239b0ae8-7aec-471d-8ff3-3248ebb35543","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:28.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"239b0ae8-7aec-471d-8ff3-3248ebb35543","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:28.067Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"239b0ae8-7aec-471d-8ff3-3248ebb35543","podName":"ttl-initializing-rmrfw-small-group-worker-88qtq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:28.067Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"239b0ae8-7aec-471d-8ff3-3248ebb35543","podName":"ttl-initializing-rmrfw-small-group-worker-88qtq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:28.067Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"239b0ae8-7aec-471d-8ff3-3248ebb35543","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:11:28.067Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"239b0ae8-7aec-471d-8ff3-3248ebb35543","seconds":300}
{"level":"info","ts":"2026-09-18T10:11:29.200Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"7a96f25c-80ff-4f6b-931a-fc7c61157545","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:29.200Z","logger":"controllers.RayJob","msg":"The RayJob has passed the preRunningDeadlineSeconds. Transition the status to `Failed`.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"7a96f25c-80ff-4f6b-931a-fc7c61157545","StartTime":"2026-09-18 10:11:23 +0000 UTC","PreRunningDeadlineSeconds":5}
{"level":"info","ts":"2026-09-18T10:11:29.200Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"7a96f25c-80ff-4f6b-931a-fc7c61157545","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.57","serviceIP":"10.132.0.57","podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-rcc8l","rayClusterName":"ttl-initializing-rmrfw","jobDeploymentStatus":"Failed","reason":"PreRunningDeadlineExceeded","message":"The RayJob has passed the preRunningDeadlineSeconds. StartTime: 2026-09-18 10:11:23 +0000 UTC. PreRunningDeadlineSeconds: 5","startTime":"2026-09-18T10:11:23Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.57","serviceIP":"10.132.0.57","podName":"ttl-initializing-rmrfw-head-mnhjd","serviceName":"ttl-initializing-rmrfw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:29.200Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"7a96f25c-80ff-4f6b-931a-fc7c61157545","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-09-18T10:11:29.219Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"5345cdfa-003a-4604-b4e7-f1c2ae159cfa","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:29.219Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"5345cdfa-003a-4604-b4e7-f1c2ae159cfa","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:11:29 +0000 UTC","Now":"2026-09-18T10:11:29.219Z","ShutdownTime":"2026-09-18T10:11:29.000Z"}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"c6abaf05-0b00-4706-a03d-63d3fa3f8de1"}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"8e01e571-8674-4728-97a4-cf0146f19115","rayCluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"fba585cf-5f47-4ba1-80a3-bcd34090b068","namespacedName":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"fba585cf-5f47-4ba1-80a3-bcd34090b068","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"fba585cf-5f47-4ba1-80a3-bcd34090b068","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"8e01e571-8674-4728-97a4-cf0146f19115","pod":"ttl-initializing-rmrfw-small-group-worker-88qtq","ip":"10.133.0.49"}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"8e01e571-8674-4728-97a4-cf0146f19115","pod":"ttl-initializing-rmrfw-head-mnhjd","ip":"10.132.0.57"}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"fba585cf-5f47-4ba1-80a3-bcd34090b068","namespace":"test-ns-ltx56","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"fba585cf-5f47-4ba1-80a3-bcd34090b068","name":"kuberay-gateway-access","namespace":"test-ns-ltx56","remainingClusters":1}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"8e01e571-8674-4728-97a4-cf0146f19115","rayCluster":"ttl-initializing-rmrfw","podIPs":["10.133.0.49","10.132.0.57"]}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"reconcileID":"d0b38d58-779c-402e-8e65-61a9db27a04a"}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"5345cdfa-003a-4604-b4e7-f1c2ae159cfa","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"5345cdfa-003a-4604-b4e7-f1c2ae159cfa","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"5345cdfa-003a-4604-b4e7-f1c2ae159cfa","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"48e30627-e49d-4269-99d2-a4f10468952d","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"48e30627-e49d-4269-99d2-a4f10468952d","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:11:29 +0000 UTC","Now":"2026-09-18T10:11:29.230Z","ShutdownTime":"2026-09-18T10:11:29.000Z"}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"48e30627-e49d-4269-99d2-a4f10468952d","RayCluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"48e30627-e49d-4269-99d2-a4f10468952d","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T10:11:29.230Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"48e30627-e49d-4269-99d2-a4f10468952d","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:29.237Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"fba585cf-5f47-4ba1-80a3-bcd34090b068","name":"test-ns-ltx56-ttl-initializing-rmrfw","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T10:11:29.244Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"fba585cf-5f47-4ba1-80a3-bcd34090b068","configMap":"kube-rbac-proxy-config-ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:29.251Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"fba585cf-5f47-4ba1-80a3-bcd34090b068","serviceAccount":"ttl-initializing-rmrfw-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:11:29.259Z","logger":"raycluster-resource","msg":"default","name":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:29.259Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:11:29.272Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"e38c42f1-f41d-4e25-a800-ec1b085fda9d"}
{"level":"info","ts":"2026-09-18T10:11:29.272Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"7c391705-7010-4d0e-80a1-e6f43976a848","namespace":"test-ns-ltx56","clusterName":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:29.273Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"54df80b6-e3cd-4477-8026-fc436d7226bf","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:29.273Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"54df80b6-e3cd-4477-8026-fc436d7226bf","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:11:29 +0000 UTC","Now":"2026-09-18T10:11:29.273Z","ShutdownTime":"2026-09-18T10:11:29.000Z"}
{"level":"info","ts":"2026-09-18T10:11:29.273Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"54df80b6-e3cd-4477-8026-fc436d7226bf","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:29.273Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"54df80b6-e3cd-4477-8026-fc436d7226bf","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T10:11:29.273Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"54df80b6-e3cd-4477-8026-fc436d7226bf","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:29.273Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"fba585cf-5f47-4ba1-80a3-bcd34090b068","cluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:29.273Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"849717f3-ccbd-4e72-b96e-0b9d9dad8612","namespacedName":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:29.273Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"849717f3-ccbd-4e72-b96e-0b9d9dad8612"}
{"level":"info","ts":"2026-09-18T10:11:29.273Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"849717f3-ccbd-4e72-b96e-0b9d9dad8612","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:11:29.273Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"849717f3-ccbd-4e72-b96e-0b9d9dad8612","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:11:29.283Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"b763a280-6fcf-4185-ae9b-35cec373a3fc"}
{"level":"info","ts":"2026-09-18T10:11:29.287Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"8db87f68-1f55-46d7-90f8-7877960aece8","namespacedName":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:29.287Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"8db87f68-1f55-46d7-90f8-7877960aece8"}
{"level":"info","ts":"2026-09-18T10:11:29.287Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"8db87f68-1f55-46d7-90f8-7877960aece8","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:11:29.287Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"8db87f68-1f55-46d7-90f8-7877960aece8","namespace":"test-ns-ltx56","authClusters":1}
{"level":"error","ts":"2026-09-18T10:11:29.295Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"7c391705-7010-4d0e-80a1-e6f43976a848","issuer":"ray-selfsigned-issuer-ttl-initializing-rmrfw","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ttl-initializing-rmrfw\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:29.303Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"49c3f383-f62d-4cf4-a7e7-b2297a8058c5"}
{"level":"info","ts":"2026-09-18T10:11:29.335Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"7c391705-7010-4d0e-80a1-e6f43976a848","errorCount":1}
{"level":"info","ts":"2026-09-18T10:11:29.335Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"7c391705-7010-4d0e-80a1-e6f43976a848"}
{"level":"error","ts":"2026-09-18T10:11:29.335Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"7c391705-7010-4d0e-80a1-e6f43976a848","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ttl-initializing-rmrfw\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:29.341Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"c02df524-2f16-45f2-8472-e56253f8ae2f","namespace":"test-ns-ltx56","clusterName":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:29.341Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-initializing-rmrfw","reconcileID":"c02df524-2f16-45f2-8472-e56253f8ae2f","namespace":"test-ns-ltx56","clusterName":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:30.175Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"79ba9629-bee8-40f8-a765-440c2967c175","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:30.175Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"79ba9629-bee8-40f8-a765-440c2967c175","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T10:11:30.190Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"79ba9629-bee8-40f8-a765-440c2967c175"}
{"level":"info","ts":"2026-09-18T10:11:30.190Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"79ba9629-bee8-40f8-a765-440c2967c175","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-09-18T10:11:30.190Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"79ba9629-bee8-40f8-a765-440c2967c175","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:11:30.190Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"79ba9629-bee8-40f8-a765-440c2967c175","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:11:30.211Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"11e2d26f-9699-4318-bb5d-83f61b420460","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:30.211Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"11e2d26f-9699-4318-bb5d-83f61b420460","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.211Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"11e2d26f-9699-4318-bb5d-83f61b420460","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:30.211Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"11e2d26f-9699-4318-bb5d-83f61b420460","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:30.222Z","logger":"raycluster-resource","msg":"default","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.222Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:11:30.241Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"11e2d26f-9699-4318-bb5d-83f61b420460","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"f54afbd1-0e3a-40e2-8c46-6f237c9522b5","namespacedName":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"f54afbd1-0e3a-40e2-8c46-6f237c9522b5","mode":"IntegratedOAuth","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"f54afbd1-0e3a-40e2-8c46-6f237c9522b5","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"f54afbd1-0e3a-40e2-8c46-6f237c9522b5","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"11e2d26f-9699-4318-bb5d-83f61b420460"}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"11e2d26f-9699-4318-bb5d-83f61b420460","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"11e2d26f-9699-4318-bb5d-83f61b420460","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"ed614931-614e-407e-be70-f9dd45d3e70e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"ed614931-614e-407e-be70-f9dd45d3e70e","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"ed614931-614e-407e-be70-f9dd45d3e70e","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"c1f03fc9-f980-4f01-8a23-932fff149b55","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"ed614931-614e-407e-be70-f9dd45d3e70e"}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"ed614931-614e-407e-be70-f9dd45d3e70e","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"ed614931-614e-407e-be70-f9dd45d3e70e","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"20e58734-e059-4481-8eed-847afa30a8d6"}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"20e58734-e059-4481-8eed-847afa30a8d6"}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"20e58734-e059-4481-8eed-847afa30a8d6"}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"56bc3a4e-b8a9-4196-b619-47fd08d11d14","rayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.242Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"56bc3a4e-b8a9-4196-b619-47fd08d11d14","rayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.252Z","logger":"raycluster-resource","msg":"default","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.252Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:11:30.274Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"c1f03fc9-f980-4f01-8a23-932fff149b55","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.274Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"fcae3112-2aaf-4e4d-916d-cebe15e10f66","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.275Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"20e58734-e059-4481-8eed-847afa30a8d6","name":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:30.275Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"20e58734-e059-4481-8eed-847afa30a8d6","error":"head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"error","ts":"2026-09-18T10:11:30.275Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"20e58734-e059-4481-8eed-847afa30a8d6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.275Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"20e58734-e059-4481-8eed-847afa30a8d6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"info","ts":"2026-09-18T10:11:30.275Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"20e58734-e059-4481-8eed-847afa30a8d6","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.275Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"20e58734-e059-4481-8eed-847afa30a8d6","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.275Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"20e58734-e059-4481-8eed-847afa30a8d6","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.275Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"20e58734-e059-4481-8eed-847afa30a8d6","name":"ttl-waiting-4wsk7","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:11:30.276Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"da96aa63-12b1-427a-9c01-87387d8209d1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:30.276Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"da96aa63-12b1-427a-9c01-87387d8209d1","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.276Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"da96aa63-12b1-427a-9c01-87387d8209d1","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:30.276Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"da96aa63-12b1-427a-9c01-87387d8209d1"}
{"level":"info","ts":"2026-09-18T10:11:30.276Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"da96aa63-12b1-427a-9c01-87387d8209d1","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:30.276Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"da96aa63-12b1-427a-9c01-87387d8209d1","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-09-18T10:11:30.284Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"20e58734-e059-4481-8eed-847afa30a8d6","name":"ttl-waiting-4wsk7","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ttl-waiting-4wsk7\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7"}}
{"level":"info","ts":"2026-09-18T10:11:30.284Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"20e58734-e059-4481-8eed-847afa30a8d6"}
{"level":"error","ts":"2026-09-18T10:11:30.284Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"20e58734-e059-4481-8eed-847afa30a8d6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.284Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ba2b4796-3bfc-4383-a56e-3071231ca277"}
{"level":"info","ts":"2026-09-18T10:11:30.284Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ba2b4796-3bfc-4383-a56e-3071231ca277"}
{"level":"info","ts":"2026-09-18T10:11:30.284Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ba2b4796-3bfc-4383-a56e-3071231ca277"}
{"level":"info","ts":"2026-09-18T10:11:30.284Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ba2b4796-3bfc-4383-a56e-3071231ca277","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:30.284Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ba2b4796-3bfc-4383-a56e-3071231ca277","error":"head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"error","ts":"2026-09-18T10:11:30.284Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ba2b4796-3bfc-4383-a56e-3071231ca277","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.284Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ba2b4796-3bfc-4383-a56e-3071231ca277","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"info","ts":"2026-09-18T10:11:30.284Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ba2b4796-3bfc-4383-a56e-3071231ca277","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.284Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ba2b4796-3bfc-4383-a56e-3071231ca277","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.284Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ba2b4796-3bfc-4383-a56e-3071231ca277","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.284Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ba2b4796-3bfc-4383-a56e-3071231ca277","name":"ttl-waiting-4wsk7","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:11:30.288Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"f54afbd1-0e3a-40e2-8c46-6f237c9522b5","name":"ttl-waiting-4wsk7-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:11:30.309Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"fcae3112-2aaf-4e4d-916d-cebe15e10f66","name":"ttl-waiting-4wsk7-head"}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ba2b4796-3bfc-4383-a56e-3071231ca277"}
{"level":"error","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ba2b4796-3bfc-4383-a56e-3071231ca277","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c7f1776e-3862-4c84-9f3c-3c400029bc59"}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c7f1776e-3862-4c84-9f3c-3c400029bc59"}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c7f1776e-3862-4c84-9f3c-3c400029bc59"}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c7f1776e-3862-4c84-9f3c-3c400029bc59","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"2b4d1037-701f-44f2-9157-7deffdc2c885","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"2b4d1037-701f-44f2-9157-7deffdc2c885","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c7f1776e-3862-4c84-9f3c-3c400029bc59","error":"head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"2b4d1037-701f-44f2-9157-7deffdc2c885","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"error","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c7f1776e-3862-4c84-9f3c-3c400029bc59","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c7f1776e-3862-4c84-9f3c-3c400029bc59","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"2b4d1037-701f-44f2-9157-7deffdc2c885"}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"2b4d1037-701f-44f2-9157-7deffdc2c885","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c7f1776e-3862-4c84-9f3c-3c400029bc59","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"2b4d1037-701f-44f2-9157-7deffdc2c885","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"2b4d1037-701f-44f2-9157-7deffdc2c885","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c7f1776e-3862-4c84-9f3c-3c400029bc59","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c7f1776e-3862-4c84-9f3c-3c400029bc59","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c7f1776e-3862-4c84-9f3c-3c400029bc59"}
{"level":"error","ts":"2026-09-18T10:11:30.310Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c7f1776e-3862-4c84-9f3c-3c400029bc59","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.317Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"f54afbd1-0e3a-40e2-8c46-6f237c9522b5","name":"test-ns-ltx56-ttl-waiting-4wsk7","operation":"created"}
{"level":"info","ts":"2026-09-18T10:11:30.321Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"5567f055-35b8-48d8-9ae9-3e9e02c6e2a4"}
{"level":"info","ts":"2026-09-18T10:11:30.321Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"5567f055-35b8-48d8-9ae9-3e9e02c6e2a4"}
{"level":"info","ts":"2026-09-18T10:11:30.321Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"5567f055-35b8-48d8-9ae9-3e9e02c6e2a4"}
{"level":"info","ts":"2026-09-18T10:11:30.321Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"5567f055-35b8-48d8-9ae9-3e9e02c6e2a4","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:30.321Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"5567f055-35b8-48d8-9ae9-3e9e02c6e2a4","error":"head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"error","ts":"2026-09-18T10:11:30.321Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"5567f055-35b8-48d8-9ae9-3e9e02c6e2a4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.321Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"5567f055-35b8-48d8-9ae9-3e9e02c6e2a4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"info","ts":"2026-09-18T10:11:30.321Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"5567f055-35b8-48d8-9ae9-3e9e02c6e2a4","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.321Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"5567f055-35b8-48d8-9ae9-3e9e02c6e2a4","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.321Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"5567f055-35b8-48d8-9ae9-3e9e02c6e2a4","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.321Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"5567f055-35b8-48d8-9ae9-3e9e02c6e2a4"}
{"level":"error","ts":"2026-09-18T10:11:30.321Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"5567f055-35b8-48d8-9ae9-3e9e02c6e2a4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.366Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"2c67fa70-fda4-442c-ba7b-43d0d85c107a"}
{"level":"info","ts":"2026-09-18T10:11:30.366Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"2c67fa70-fda4-442c-ba7b-43d0d85c107a"}
{"level":"info","ts":"2026-09-18T10:11:30.366Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"2c67fa70-fda4-442c-ba7b-43d0d85c107a"}
{"level":"info","ts":"2026-09-18T10:11:30.366Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"2c67fa70-fda4-442c-ba7b-43d0d85c107a","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:30.366Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"2c67fa70-fda4-442c-ba7b-43d0d85c107a","error":"head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"info","ts":"2026-09-18T10:11:30.366Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"56bc3a4e-b8a9-4196-b619-47fd08d11d14","rayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.366Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"56bc3a4e-b8a9-4196-b619-47fd08d11d14","rayCluster":"ttl-waiting-4wsk7"}
{"level":"error","ts":"2026-09-18T10:11:30.366Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"2c67fa70-fda4-442c-ba7b-43d0d85c107a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.366Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"2c67fa70-fda4-442c-ba7b-43d0d85c107a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"info","ts":"2026-09-18T10:11:30.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"2c67fa70-fda4-442c-ba7b-43d0d85c107a","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"2c67fa70-fda4-442c-ba7b-43d0d85c107a","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"2c67fa70-fda4-442c-ba7b-43d0d85c107a","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.366Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"2c67fa70-fda4-442c-ba7b-43d0d85c107a"}
{"level":"error","ts":"2026-09-18T10:11:30.366Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"2c67fa70-fda4-442c-ba7b-43d0d85c107a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.366Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"f54afbd1-0e3a-40e2-8c46-6f237c9522b5","name":"kube-rbac-proxy-config-ttl-waiting-4wsk7","operation":"created"}
{"level":"info","ts":"2026-09-18T10:11:30.366Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e01c33d4-5b4c-410c-b625-7d04ac15df79","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:30.366Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e01c33d4-5b4c-410c-b625-7d04ac15df79","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.366Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e01c33d4-5b4c-410c-b625-7d04ac15df79","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:30.367Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e01c33d4-5b4c-410c-b625-7d04ac15df79"}
{"level":"info","ts":"2026-09-18T10:11:30.367Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e01c33d4-5b4c-410c-b625-7d04ac15df79","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:30.367Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e01c33d4-5b4c-410c-b625-7d04ac15df79","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:30.378Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"fcae3112-2aaf-4e4d-916d-cebe15e10f66","name":"ttl-waiting-4wsk7-workers"}
{"level":"info","ts":"2026-09-18T10:11:30.378Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"fcae3112-2aaf-4e4d-916d-cebe15e10f66","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.378Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"884b9094-cd57-4faf-b2a5-3c3bd729fba2","cluster":"ttl-waiting-4wsk7"}
{"level":"error","ts":"2026-09-18T10:11:30.392Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"f54afbd1-0e3a-40e2-8c46-6f237c9522b5","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ttl-waiting-4wsk7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.392Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"f54afbd1-0e3a-40e2-8c46-6f237c9522b5"}
{"level":"error","ts":"2026-09-18T10:11:30.392Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"f54afbd1-0e3a-40e2-8c46-6f237c9522b5","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ttl-waiting-4wsk7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.392Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"3894e8de-bea6-4182-b892-17fa9f3f130d","namespacedName":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:30.392Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"3894e8de-bea6-4182-b892-17fa9f3f130d","mode":"IntegratedOAuth","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.392Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"3894e8de-bea6-4182-b892-17fa9f3f130d","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.414Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"884b9094-cd57-4faf-b2a5-3c3bd729fba2","name":"ttl-waiting-4wsk7-head"}
{"level":"info","ts":"2026-09-18T10:11:30.416Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"06c80a4f-93d8-4101-9457-cf0d759cb266","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:30.416Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"06c80a4f-93d8-4101-9457-cf0d759cb266","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.416Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"06c80a4f-93d8-4101-9457-cf0d759cb266","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:30.416Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"06c80a4f-93d8-4101-9457-cf0d759cb266"}
{"level":"info","ts":"2026-09-18T10:11:30.416Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"06c80a4f-93d8-4101-9457-cf0d759cb266","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:30.416Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"06c80a4f-93d8-4101-9457-cf0d759cb266","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:30.416Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"06c80a4f-93d8-4101-9457-cf0d759cb266","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:11:30.416Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"3894e8de-bea6-4182-b892-17fa9f3f130d","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.416Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"feb2589a-e483-4554-9150-35a3b0bdbeb9","namespacedName":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:30.416Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"feb2589a-e483-4554-9150-35a3b0bdbeb9","mode":"IntegratedOAuth","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.416Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"feb2589a-e483-4554-9150-35a3b0bdbeb9","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.424Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"56bc3a4e-b8a9-4196-b619-47fd08d11d14","rayCluster":"ttl-waiting-4wsk7","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T10:11:30.424Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"56bc3a4e-b8a9-4196-b619-47fd08d11d14","rayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.443Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"feb2589a-e483-4554-9150-35a3b0bdbeb9","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.444Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"2ca868c5-339d-49a5-ba55-7e8d88dd7d8d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:30.444Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"2ca868c5-339d-49a5-ba55-7e8d88dd7d8d","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.444Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"2ca868c5-339d-49a5-ba55-7e8d88dd7d8d","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:30.444Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"2ca868c5-339d-49a5-ba55-7e8d88dd7d8d"}
{"level":"info","ts":"2026-09-18T10:11:30.444Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"2ca868c5-339d-49a5-ba55-7e8d88dd7d8d","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:30.444Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"2ca868c5-339d-49a5-ba55-7e8d88dd7d8d","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:30.447Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"22a305cb-04cc-4f5b-9bfa-51589f6d80f7"}
{"level":"info","ts":"2026-09-18T10:11:30.447Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"22a305cb-04cc-4f5b-9bfa-51589f6d80f7"}
{"level":"info","ts":"2026-09-18T10:11:30.447Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"22a305cb-04cc-4f5b-9bfa-51589f6d80f7"}
{"level":"info","ts":"2026-09-18T10:11:30.447Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"22a305cb-04cc-4f5b-9bfa-51589f6d80f7","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:30.447Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"22a305cb-04cc-4f5b-9bfa-51589f6d80f7","error":"head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"error","ts":"2026-09-18T10:11:30.447Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"22a305cb-04cc-4f5b-9bfa-51589f6d80f7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.447Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"22a305cb-04cc-4f5b-9bfa-51589f6d80f7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"info","ts":"2026-09-18T10:11:30.447Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"22a305cb-04cc-4f5b-9bfa-51589f6d80f7","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.447Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"22a305cb-04cc-4f5b-9bfa-51589f6d80f7","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.447Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"22a305cb-04cc-4f5b-9bfa-51589f6d80f7","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.447Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"22a305cb-04cc-4f5b-9bfa-51589f6d80f7"}
{"level":"error","ts":"2026-09-18T10:11:30.447Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"22a305cb-04cc-4f5b-9bfa-51589f6d80f7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.448Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"884b9094-cd57-4faf-b2a5-3c3bd729fba2","name":"ttl-waiting-4wsk7-workers"}
{"level":"info","ts":"2026-09-18T10:11:30.448Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"884b9094-cd57-4faf-b2a5-3c3bd729fba2","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.448Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"7b6a3e74-175f-420b-ab1f-94c79021e26c","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.466Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"56bc3a4e-b8a9-4196-b619-47fd08d11d14","rayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.472Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"7b6a3e74-175f-420b-ab1f-94c79021e26c","name":"ttl-waiting-4wsk7-head"}
{"level":"info","ts":"2026-09-18T10:11:30.506Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"7b6a3e74-175f-420b-ab1f-94c79021e26c","name":"ttl-waiting-4wsk7-workers"}
{"level":"info","ts":"2026-09-18T10:11:30.506Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"7b6a3e74-175f-420b-ab1f-94c79021e26c","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.511Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"56bc3a4e-b8a9-4196-b619-47fd08d11d14","rayCluster":"ttl-waiting-4wsk7","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:11:30.526Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"56bc3a4e-b8a9-4196-b619-47fd08d11d14","rayCluster":"ttl-waiting-4wsk7","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:11:30.526Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"56bc3a4e-b8a9-4196-b619-47fd08d11d14"}
{"level":"info","ts":"2026-09-18T10:11:30.526Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"79a1ce7c-1620-4528-8794-330c46d08631","rayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:30.526Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"79a1ce7c-1620-4528-8794-330c46d08631"}
{"level":"info","ts":"2026-09-18T10:11:30.607Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"12698d24-65df-42d1-8767-17941f0bebc7"}
{"level":"info","ts":"2026-09-18T10:11:30.607Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"12698d24-65df-42d1-8767-17941f0bebc7"}
{"level":"info","ts":"2026-09-18T10:11:30.607Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"12698d24-65df-42d1-8767-17941f0bebc7"}
{"level":"info","ts":"2026-09-18T10:11:30.607Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"12698d24-65df-42d1-8767-17941f0bebc7","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:30.607Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"12698d24-65df-42d1-8767-17941f0bebc7","error":"head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"error","ts":"2026-09-18T10:11:30.607Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"12698d24-65df-42d1-8767-17941f0bebc7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.607Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"12698d24-65df-42d1-8767-17941f0bebc7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"info","ts":"2026-09-18T10:11:30.607Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"12698d24-65df-42d1-8767-17941f0bebc7","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.608Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"12698d24-65df-42d1-8767-17941f0bebc7","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.608Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"12698d24-65df-42d1-8767-17941f0bebc7","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.608Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"12698d24-65df-42d1-8767-17941f0bebc7"}
{"level":"error","ts":"2026-09-18T10:11:30.608Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"12698d24-65df-42d1-8767-17941f0bebc7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.928Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"61eb88f7-0536-45c8-9449-730b7f23ec61"}
{"level":"info","ts":"2026-09-18T10:11:30.928Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"61eb88f7-0536-45c8-9449-730b7f23ec61"}
{"level":"info","ts":"2026-09-18T10:11:30.928Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"61eb88f7-0536-45c8-9449-730b7f23ec61"}
{"level":"info","ts":"2026-09-18T10:11:30.929Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"61eb88f7-0536-45c8-9449-730b7f23ec61","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:30.929Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"61eb88f7-0536-45c8-9449-730b7f23ec61","error":"head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"error","ts":"2026-09-18T10:11:30.929Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"61eb88f7-0536-45c8-9449-730b7f23ec61","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:30.929Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"61eb88f7-0536-45c8-9449-730b7f23ec61","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"info","ts":"2026-09-18T10:11:30.929Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"61eb88f7-0536-45c8-9449-730b7f23ec61","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.929Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"61eb88f7-0536-45c8-9449-730b7f23ec61","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.929Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"61eb88f7-0536-45c8-9449-730b7f23ec61","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:30.929Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"61eb88f7-0536-45c8-9449-730b7f23ec61"}
{"level":"error","ts":"2026-09-18T10:11:30.929Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"61eb88f7-0536-45c8-9449-730b7f23ec61","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:31.570Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4364ac39-0815-4972-876a-c42ede05d33a"}
{"level":"info","ts":"2026-09-18T10:11:31.570Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4364ac39-0815-4972-876a-c42ede05d33a"}
{"level":"info","ts":"2026-09-18T10:11:31.570Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4364ac39-0815-4972-876a-c42ede05d33a"}
{"level":"info","ts":"2026-09-18T10:11:31.570Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4364ac39-0815-4972-876a-c42ede05d33a","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:31.570Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4364ac39-0815-4972-876a-c42ede05d33a","error":"head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"error","ts":"2026-09-18T10:11:31.570Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4364ac39-0815-4972-876a-c42ede05d33a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:31.570Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4364ac39-0815-4972-876a-c42ede05d33a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found"}
{"level":"info","ts":"2026-09-18T10:11:31.570Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4364ac39-0815-4972-876a-c42ede05d33a","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:31.570Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4364ac39-0815-4972-876a-c42ede05d33a","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:31.570Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4364ac39-0815-4972-876a-c42ede05d33a","filter labels":["test-ns-ltx56",{"ray.io/cluster":"ttl-waiting-4wsk7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:11:31.570Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4364ac39-0815-4972-876a-c42ede05d33a"}
{"level":"error","ts":"2026-09-18T10:11:31.570Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4364ac39-0815-4972-876a-c42ede05d33a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ttl-waiting-4wsk7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:32.218Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"a3311e36-2866-4229-a7f9-9b6c5998eb08","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:11:32.218Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"a3311e36-2866-4229-a7f9-9b6c5998eb08","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:11:29 +0000 UTC","Now":"2026-09-18T10:11:32.218Z","ShutdownTime":"2026-09-18T10:11:29.000Z"}
{"level":"info","ts":"2026-09-18T10:11:32.218Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"a3311e36-2866-4229-a7f9-9b6c5998eb08","RayCluster":{"name":"ttl-initializing-rmrfw","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:32.218Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"a3311e36-2866-4229-a7f9-9b6c5998eb08","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T10:11:32.218Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"a3311e36-2866-4229-a7f9-9b6c5998eb08","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"ttl-initializing-rmrfw"}
{"level":"info","ts":"2026-09-18T10:11:32.851Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c"}
{"level":"info","ts":"2026-09-18T10:11:32.851Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c"}
{"level":"info","ts":"2026-09-18T10:11:32.851Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c"}
{"level":"info","ts":"2026-09-18T10:11:32.851Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:32.851Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c"}
{"level":"info","ts":"2026-09-18T10:11:32.851Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:11:32.851Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c"}
{"level":"info","ts":"2026-09-18T10:11:32.851Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","mode":"IntegratedOAuth","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:32.851Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","cluster":"ttl-waiting-4wsk7","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:11:32.851Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","cluster":"ttl-waiting-4wsk7","authType":"IntegratedOAuth","serviceAccount":"ttl-waiting-4wsk7-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T10:11:32.851Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ttl-waiting-4wsk7","ray.io/group":"headgroup","ray.io/identifier":"ttl-waiting-4wsk7-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:11:32.851Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T10:11:32.851Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:11:32.851Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:11:32.851Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","enabled":true}
{"level":"info","ts":"2026-09-18T10:11:32.874Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","name":"ttl-waiting-4wsk7-head-q4ncm"}
{"level":"info","ts":"2026-09-18T10:11:32.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:32.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:32.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:11:32.874Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:11:32.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:32.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:11:32.874Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c"}
{"level":"info","ts":"2026-09-18T10:11:32.874Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","nodeType":"worker","rayStartParams":{"address":"ttl-waiting-4wsk7-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:11:32.874Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","rayStartCmd":"ray start  --address=ttl-waiting-4wsk7-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:11:32.874Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=ttl-waiting-4wsk7-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:11:32.874Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","enabled":true}
{"level":"info","ts":"2026-09-18T10:11:32.897Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","name":"ttl-waiting-4wsk7-small-group-worker-vfzdt"}
{"level":"info","ts":"2026-09-18T10:11:32.897Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:32.897Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"fb021c0b-9766-4a76-a6c1-40ac9363238c","name":"ttl-waiting-4wsk7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:11:32.932Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8"}
{"level":"info","ts":"2026-09-18T10:11:32.932Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8"}
{"level":"info","ts":"2026-09-18T10:11:32.932Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8"}
{"level":"info","ts":"2026-09-18T10:11:32.932Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:32.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:32.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:32.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:32.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:32.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:32.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:32.932Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"6430d3c8-1f33-4c2f-95dc-8d4fe05cda31","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:32.932Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"6430d3c8-1f33-4c2f-95dc-8d4fe05cda31","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:32.932Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"6430d3c8-1f33-4c2f-95dc-8d4fe05cda31","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:32.932Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"6430d3c8-1f33-4c2f-95dc-8d4fe05cda31"}
{"level":"info","ts":"2026-09-18T10:11:32.933Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"6430d3c8-1f33-4c2f-95dc-8d4fe05cda31","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:32.933Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"6430d3c8-1f33-4c2f-95dc-8d4fe05cda31","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:32.933Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"6430d3c8-1f33-4c2f-95dc-8d4fe05cda31","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:11:32.933Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"6334706b-e4a5-4e23-b57d-b7668e2fdb6b","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:32.933Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"40aead60-b533-4249-83ab-58cac60f13e9","rayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:32.932Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:32.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:32.966Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"40aead60-b533-4249-83ab-58cac60f13e9","rayCluster":"ttl-waiting-4wsk7","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:11:32.960Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"6334706b-e4a5-4e23-b57d-b7668e2fdb6b","name":"ttl-waiting-4wsk7-head"}
{"level":"info","ts":"2026-09-18T10:11:32.966Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:32.966Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:32.966Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8","name":"ttl-waiting-4wsk7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:11:32.962Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"1a6193d0-0b8a-4400-b31b-7c1b1b6a7e75","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:32.966Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"1a6193d0-0b8a-4400-b31b-7c1b1b6a7e75","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:32.966Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"1a6193d0-0b8a-4400-b31b-7c1b1b6a7e75","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:32.966Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"1a6193d0-0b8a-4400-b31b-7c1b1b6a7e75"}
{"level":"info","ts":"2026-09-18T10:11:32.966Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"1a6193d0-0b8a-4400-b31b-7c1b1b6a7e75","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:32.966Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"1a6193d0-0b8a-4400-b31b-7c1b1b6a7e75","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8","name":"ttl-waiting-4wsk7","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ttl-waiting-4wsk7\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7"}}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8"}
{"level":"error","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"bde02d39-50d2-45c8-801a-1a5da5ec5db8","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ttl-waiting-4wsk7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"46b607c0-90d1-41e5-a53b-f2543cb828b4"}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"46b607c0-90d1-41e5-a53b-f2543cb828b4"}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"46b607c0-90d1-41e5-a53b-f2543cb828b4"}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"46b607c0-90d1-41e5-a53b-f2543cb828b4","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"46b607c0-90d1-41e5-a53b-f2543cb828b4","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"46b607c0-90d1-41e5-a53b-f2543cb828b4","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"46b607c0-90d1-41e5-a53b-f2543cb828b4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"46b607c0-90d1-41e5-a53b-f2543cb828b4","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"46b607c0-90d1-41e5-a53b-f2543cb828b4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"46b607c0-90d1-41e5-a53b-f2543cb828b4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"46b607c0-90d1-41e5-a53b-f2543cb828b4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"46b607c0-90d1-41e5-a53b-f2543cb828b4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"46b607c0-90d1-41e5-a53b-f2543cb828b4","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"46b607c0-90d1-41e5-a53b-f2543cb828b4","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:32.981Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"46b607c0-90d1-41e5-a53b-f2543cb828b4","name":"ttl-waiting-4wsk7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:11:32.983Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"6334706b-e4a5-4e23-b57d-b7668e2fdb6b","name":"ttl-waiting-4wsk7-workers"}
{"level":"info","ts":"2026-09-18T10:11:32.984Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"6334706b-e4a5-4e23-b57d-b7668e2fdb6b","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:33.048Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0"}
{"level":"info","ts":"2026-09-18T10:11:33.048Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0"}
{"level":"info","ts":"2026-09-18T10:11:33.048Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0"}
{"level":"info","ts":"2026-09-18T10:11:33.048Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:33.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:33.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:33.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:33.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:33.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:33.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:33.048Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:33.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:33.048Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:33.048Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:33.048Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0","name":"ttl-waiting-4wsk7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:11:33.054Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"2378e3e8-8350-4f0a-a68c-6aa4dc55bcff","rayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:33.054Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"696f5005-004e-4a83-8f74-f60e43858066","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:33.055Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"d5dcb661-8b4e-4846-a706-e7fccf0537f9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:33.055Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"d5dcb661-8b4e-4846-a706-e7fccf0537f9","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:33.055Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"d5dcb661-8b4e-4846-a706-e7fccf0537f9","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:33.055Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"d5dcb661-8b4e-4846-a706-e7fccf0537f9"}
{"level":"info","ts":"2026-09-18T10:11:33.055Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"d5dcb661-8b4e-4846-a706-e7fccf0537f9","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:33.055Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"d5dcb661-8b4e-4846-a706-e7fccf0537f9","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:33.055Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"d5dcb661-8b4e-4846-a706-e7fccf0537f9","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:11:33.066Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"2378e3e8-8350-4f0a-a68c-6aa4dc55bcff","rayCluster":"ttl-waiting-4wsk7","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:11:33.085Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0","name":"ttl-waiting-4wsk7","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ttl-waiting-4wsk7\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7"}}
{"level":"info","ts":"2026-09-18T10:11:33.085Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0"}
{"level":"error","ts":"2026-09-18T10:11:33.085Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"0384e632-0866-4ba4-86ed-8b4cc3740fd0","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ttl-waiting-4wsk7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"bb215e12-4587-403f-923b-8db3eafafc17","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"bb215e12-4587-403f-923b-8db3eafafc17","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"bb215e12-4587-403f-923b-8db3eafafc17","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ebf3d7b0-65f2-4bd8-8bfe-66ee5c05878f"}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ebf3d7b0-65f2-4bd8-8bfe-66ee5c05878f"}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ebf3d7b0-65f2-4bd8-8bfe-66ee5c05878f"}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ebf3d7b0-65f2-4bd8-8bfe-66ee5c05878f","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"bb215e12-4587-403f-923b-8db3eafafc17"}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"bb215e12-4587-403f-923b-8db3eafafc17","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"bb215e12-4587-403f-923b-8db3eafafc17","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ebf3d7b0-65f2-4bd8-8bfe-66ee5c05878f","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ebf3d7b0-65f2-4bd8-8bfe-66ee5c05878f","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ebf3d7b0-65f2-4bd8-8bfe-66ee5c05878f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ebf3d7b0-65f2-4bd8-8bfe-66ee5c05878f","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ebf3d7b0-65f2-4bd8-8bfe-66ee5c05878f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ebf3d7b0-65f2-4bd8-8bfe-66ee5c05878f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ebf3d7b0-65f2-4bd8-8bfe-66ee5c05878f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ebf3d7b0-65f2-4bd8-8bfe-66ee5c05878f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:33.090Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ebf3d7b0-65f2-4bd8-8bfe-66ee5c05878f","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:33.091Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ebf3d7b0-65f2-4bd8-8bfe-66ee5c05878f","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:33.091Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ebf3d7b0-65f2-4bd8-8bfe-66ee5c05878f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:11:33.091Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ebf3d7b0-65f2-4bd8-8bfe-66ee5c05878f","seconds":300}
{"level":"info","ts":"2026-09-18T10:11:33.094Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"696f5005-004e-4a83-8f74-f60e43858066","name":"ttl-waiting-4wsk7-head"}
{"level":"info","ts":"2026-09-18T10:11:33.110Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"696f5005-004e-4a83-8f74-f60e43858066","name":"ttl-waiting-4wsk7-workers"}
{"level":"info","ts":"2026-09-18T10:11:33.110Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"696f5005-004e-4a83-8f74-f60e43858066","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:33.120Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"9a2f862c-f484-4ad2-9cf6-d35577c4457e"}
{"level":"info","ts":"2026-09-18T10:11:33.120Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"9a2f862c-f484-4ad2-9cf6-d35577c4457e"}
{"level":"info","ts":"2026-09-18T10:11:33.120Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"9a2f862c-f484-4ad2-9cf6-d35577c4457e"}
{"level":"info","ts":"2026-09-18T10:11:33.120Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"9a2f862c-f484-4ad2-9cf6-d35577c4457e","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:33.120Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"9a2f862c-f484-4ad2-9cf6-d35577c4457e","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:33.120Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"9a2f862c-f484-4ad2-9cf6-d35577c4457e","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:33.120Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"9a2f862c-f484-4ad2-9cf6-d35577c4457e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:33.120Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"9a2f862c-f484-4ad2-9cf6-d35577c4457e","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:33.120Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"9a2f862c-f484-4ad2-9cf6-d35577c4457e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:33.120Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"9a2f862c-f484-4ad2-9cf6-d35577c4457e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:33.120Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"9a2f862c-f484-4ad2-9cf6-d35577c4457e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:33.120Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"9a2f862c-f484-4ad2-9cf6-d35577c4457e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:33.120Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"9a2f862c-f484-4ad2-9cf6-d35577c4457e","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:33.120Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"9a2f862c-f484-4ad2-9cf6-d35577c4457e","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:33.120Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"9a2f862c-f484-4ad2-9cf6-d35577c4457e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:11:33.120Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"9a2f862c-f484-4ad2-9cf6-d35577c4457e","seconds":300}
{"level":"info","ts":"2026-09-18T10:11:33.212Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"dc7b028e-778d-41d3-b8ba-d5553cda83a0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:33.212Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"dc7b028e-778d-41d3-b8ba-d5553cda83a0","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:33.212Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"dc7b028e-778d-41d3-b8ba-d5553cda83a0","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:33.212Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"dc7b028e-778d-41d3-b8ba-d5553cda83a0"}
{"level":"info","ts":"2026-09-18T10:11:33.212Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"dc7b028e-778d-41d3-b8ba-d5553cda83a0","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:33.212Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"dc7b028e-778d-41d3-b8ba-d5553cda83a0","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:33.493Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4b7541b1-0ff7-4d9a-b0e6-84752e814f81"}
{"level":"info","ts":"2026-09-18T10:11:33.493Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4b7541b1-0ff7-4d9a-b0e6-84752e814f81"}
{"level":"info","ts":"2026-09-18T10:11:33.493Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4b7541b1-0ff7-4d9a-b0e6-84752e814f81"}
{"level":"info","ts":"2026-09-18T10:11:33.493Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4b7541b1-0ff7-4d9a-b0e6-84752e814f81","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:33.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4b7541b1-0ff7-4d9a-b0e6-84752e814f81","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:33.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4b7541b1-0ff7-4d9a-b0e6-84752e814f81","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:33.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4b7541b1-0ff7-4d9a-b0e6-84752e814f81","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:33.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4b7541b1-0ff7-4d9a-b0e6-84752e814f81","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:33.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4b7541b1-0ff7-4d9a-b0e6-84752e814f81","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:33.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4b7541b1-0ff7-4d9a-b0e6-84752e814f81","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:33.494Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4b7541b1-0ff7-4d9a-b0e6-84752e814f81","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:33.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4b7541b1-0ff7-4d9a-b0e6-84752e814f81","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:33.494Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4b7541b1-0ff7-4d9a-b0e6-84752e814f81","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:33.494Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4b7541b1-0ff7-4d9a-b0e6-84752e814f81","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:33.494Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4b7541b1-0ff7-4d9a-b0e6-84752e814f81","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:11:33.494Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"4b7541b1-0ff7-4d9a-b0e6-84752e814f81","seconds":300}
{"level":"info","ts":"2026-09-18T10:11:33.532Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6fa61cb1-503c-4892-b50b-7f54c2c3956c"}
{"level":"info","ts":"2026-09-18T10:11:33.532Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6fa61cb1-503c-4892-b50b-7f54c2c3956c"}
{"level":"info","ts":"2026-09-18T10:11:33.532Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6fa61cb1-503c-4892-b50b-7f54c2c3956c"}
{"level":"info","ts":"2026-09-18T10:11:33.532Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6fa61cb1-503c-4892-b50b-7f54c2c3956c","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:33.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6fa61cb1-503c-4892-b50b-7f54c2c3956c","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:33.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6fa61cb1-503c-4892-b50b-7f54c2c3956c","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:33.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6fa61cb1-503c-4892-b50b-7f54c2c3956c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:33.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6fa61cb1-503c-4892-b50b-7f54c2c3956c","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:33.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6fa61cb1-503c-4892-b50b-7f54c2c3956c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:33.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6fa61cb1-503c-4892-b50b-7f54c2c3956c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:33.532Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6fa61cb1-503c-4892-b50b-7f54c2c3956c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:33.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6fa61cb1-503c-4892-b50b-7f54c2c3956c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:33.532Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6fa61cb1-503c-4892-b50b-7f54c2c3956c","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:33.532Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6fa61cb1-503c-4892-b50b-7f54c2c3956c","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:33.533Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6fa61cb1-503c-4892-b50b-7f54c2c3956c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:11:33.533Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6fa61cb1-503c-4892-b50b-7f54c2c3956c","seconds":300}
{"level":"info","ts":"2026-09-18T10:11:33.672Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c320715b-f2e5-4fe8-8885-3c7d001c15f6"}
{"level":"info","ts":"2026-09-18T10:11:33.672Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c320715b-f2e5-4fe8-8885-3c7d001c15f6"}
{"level":"info","ts":"2026-09-18T10:11:33.672Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c320715b-f2e5-4fe8-8885-3c7d001c15f6"}
{"level":"info","ts":"2026-09-18T10:11:33.672Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c320715b-f2e5-4fe8-8885-3c7d001c15f6","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:33.672Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c320715b-f2e5-4fe8-8885-3c7d001c15f6","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:33.672Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c320715b-f2e5-4fe8-8885-3c7d001c15f6","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:33.672Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c320715b-f2e5-4fe8-8885-3c7d001c15f6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:33.673Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c320715b-f2e5-4fe8-8885-3c7d001c15f6","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:33.673Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c320715b-f2e5-4fe8-8885-3c7d001c15f6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:33.673Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c320715b-f2e5-4fe8-8885-3c7d001c15f6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:33.673Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c320715b-f2e5-4fe8-8885-3c7d001c15f6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:33.673Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c320715b-f2e5-4fe8-8885-3c7d001c15f6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:33.673Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c320715b-f2e5-4fe8-8885-3c7d001c15f6","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:33.673Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c320715b-f2e5-4fe8-8885-3c7d001c15f6","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:33.673Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"c320715b-f2e5-4fe8-8885-3c7d001c15f6","name":"ttl-waiting-4wsk7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:11:33.709Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"918de15d-6724-401b-b076-2c7221f79dfb","rayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:33.709Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"918de15d-6724-401b-b076-2c7221f79dfb","pod":"ttl-waiting-4wsk7-head-q4ncm","ip":"10.132.0.58"}
{"level":"info","ts":"2026-09-18T10:11:33.709Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"5f539805-e232-48a8-812b-34972ef12c5f","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:33.709Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9a35127c-a498-4485-8cbb-37ba97c03e97","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:33.709Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9a35127c-a498-4485-8cbb-37ba97c03e97","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:33.709Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9a35127c-a498-4485-8cbb-37ba97c03e97","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:33.709Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9a35127c-a498-4485-8cbb-37ba97c03e97"}
{"level":"info","ts":"2026-09-18T10:11:33.709Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9a35127c-a498-4485-8cbb-37ba97c03e97","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:33.709Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9a35127c-a498-4485-8cbb-37ba97c03e97","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:33.709Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9a35127c-a498-4485-8cbb-37ba97c03e97","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:11:33.728Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"8c073ae1-8282-4d54-8410-8119fe18a789","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:33.728Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"8c073ae1-8282-4d54-8410-8119fe18a789","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:33.728Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"8c073ae1-8282-4d54-8410-8119fe18a789","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:33.729Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"8c073ae1-8282-4d54-8410-8119fe18a789"}
{"level":"info","ts":"2026-09-18T10:11:33.729Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"8c073ae1-8282-4d54-8410-8119fe18a789","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:33.729Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"8c073ae1-8282-4d54-8410-8119fe18a789","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:33.732Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"918de15d-6724-401b-b076-2c7221f79dfb","certificate":"ray-head-cert-ttl-waiting-4wsk7","dnsNames":["ttl-waiting-4wsk7-head-svc","localhost","ttl-waiting-4wsk7-head-svc.test-ns-ltx56.svc","ttl-waiting-4wsk7-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.58","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:11:33.751Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"5f539805-e232-48a8-812b-34972ef12c5f","name":"ttl-waiting-4wsk7-head"}
{"level":"info","ts":"2026-09-18T10:11:33.760Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"918de15d-6724-401b-b076-2c7221f79dfb","certificate":"ray-worker-cert-ttl-waiting-4wsk7","dnsNames":["ttl-waiting-4wsk7-worker-svc","localhost","ttl-waiting-4wsk7-small-group","ttl-waiting-4wsk7-worker-svc.test-ns-ltx56.svc","ttl-waiting-4wsk7-worker-svc.test-ns-ltx56.svc.cluster.local","*.ttl-waiting-4wsk7-worker-svc.test-ns-ltx56.svc","*.ttl-waiting-4wsk7-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.58","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:11:33.760Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"918de15d-6724-401b-b076-2c7221f79dfb","rayCluster":"ttl-waiting-4wsk7","podIPs":["10.132.0.58"]}
{"level":"info","ts":"2026-09-18T10:11:33.780Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"5f539805-e232-48a8-812b-34972ef12c5f","name":"ttl-waiting-4wsk7-workers"}
{"level":"info","ts":"2026-09-18T10:11:33.780Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"5f539805-e232-48a8-812b-34972ef12c5f","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:33.799Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ceaac411-65a6-4791-8638-e349a92d9d25"}
{"level":"info","ts":"2026-09-18T10:11:33.799Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ceaac411-65a6-4791-8638-e349a92d9d25"}
{"level":"info","ts":"2026-09-18T10:11:33.799Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ceaac411-65a6-4791-8638-e349a92d9d25"}
{"level":"info","ts":"2026-09-18T10:11:33.799Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ceaac411-65a6-4791-8638-e349a92d9d25","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:33.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ceaac411-65a6-4791-8638-e349a92d9d25","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:33.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ceaac411-65a6-4791-8638-e349a92d9d25","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:33.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ceaac411-65a6-4791-8638-e349a92d9d25","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:33.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ceaac411-65a6-4791-8638-e349a92d9d25","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:33.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ceaac411-65a6-4791-8638-e349a92d9d25","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:33.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ceaac411-65a6-4791-8638-e349a92d9d25","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:33.799Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ceaac411-65a6-4791-8638-e349a92d9d25","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:33.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ceaac411-65a6-4791-8638-e349a92d9d25","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:33.799Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ceaac411-65a6-4791-8638-e349a92d9d25","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:33.799Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ceaac411-65a6-4791-8638-e349a92d9d25","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:33.800Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ceaac411-65a6-4791-8638-e349a92d9d25","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:11:33.800Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ceaac411-65a6-4791-8638-e349a92d9d25","seconds":300}
{"level":"info","ts":"2026-09-18T10:11:35.710Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"7812baf8-1a68-4227-993c-106789e4b86d"}
{"level":"info","ts":"2026-09-18T10:11:35.710Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"7812baf8-1a68-4227-993c-106789e4b86d"}
{"level":"info","ts":"2026-09-18T10:11:35.710Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"7812baf8-1a68-4227-993c-106789e4b86d"}
{"level":"info","ts":"2026-09-18T10:11:35.710Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"7812baf8-1a68-4227-993c-106789e4b86d","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:35.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"7812baf8-1a68-4227-993c-106789e4b86d","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:35.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"7812baf8-1a68-4227-993c-106789e4b86d","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:35.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"7812baf8-1a68-4227-993c-106789e4b86d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:35.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"7812baf8-1a68-4227-993c-106789e4b86d","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:35.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"7812baf8-1a68-4227-993c-106789e4b86d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:35.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"7812baf8-1a68-4227-993c-106789e4b86d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:35.710Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"7812baf8-1a68-4227-993c-106789e4b86d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:35.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"7812baf8-1a68-4227-993c-106789e4b86d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:35.710Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"7812baf8-1a68-4227-993c-106789e4b86d","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:35.710Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"7812baf8-1a68-4227-993c-106789e4b86d","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:35.710Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"7812baf8-1a68-4227-993c-106789e4b86d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:11:35.710Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"7812baf8-1a68-4227-993c-106789e4b86d","seconds":300}
{"level":"info","ts":"2026-09-18T10:11:36.212Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"68888f59-71d8-49fd-b87c-0bd99e0ad2fb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:36.212Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"68888f59-71d8-49fd-b87c-0bd99e0ad2fb","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:36.212Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"68888f59-71d8-49fd-b87c-0bd99e0ad2fb","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:36.213Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"68888f59-71d8-49fd-b87c-0bd99e0ad2fb"}
{"level":"info","ts":"2026-09-18T10:11:36.213Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"68888f59-71d8-49fd-b87c-0bd99e0ad2fb","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:36.213Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"68888f59-71d8-49fd-b87c-0bd99e0ad2fb","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:38.712Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b15c744f-bf30-4d65-aa37-5e29201eb1bd"}
{"level":"info","ts":"2026-09-18T10:11:38.712Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b15c744f-bf30-4d65-aa37-5e29201eb1bd"}
{"level":"info","ts":"2026-09-18T10:11:38.712Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b15c744f-bf30-4d65-aa37-5e29201eb1bd"}
{"level":"info","ts":"2026-09-18T10:11:38.712Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b15c744f-bf30-4d65-aa37-5e29201eb1bd","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:38.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b15c744f-bf30-4d65-aa37-5e29201eb1bd","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:38.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b15c744f-bf30-4d65-aa37-5e29201eb1bd","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:38.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b15c744f-bf30-4d65-aa37-5e29201eb1bd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:38.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b15c744f-bf30-4d65-aa37-5e29201eb1bd","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:38.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b15c744f-bf30-4d65-aa37-5e29201eb1bd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:38.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b15c744f-bf30-4d65-aa37-5e29201eb1bd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:38.712Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b15c744f-bf30-4d65-aa37-5e29201eb1bd","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:38.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b15c744f-bf30-4d65-aa37-5e29201eb1bd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:38.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b15c744f-bf30-4d65-aa37-5e29201eb1bd","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:38.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b15c744f-bf30-4d65-aa37-5e29201eb1bd","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:38.712Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b15c744f-bf30-4d65-aa37-5e29201eb1bd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:11:38.712Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b15c744f-bf30-4d65-aa37-5e29201eb1bd","seconds":300}
{"level":"info","ts":"2026-09-18T10:11:39.213Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"64ea071f-9748-4847-a81b-3f7db4382128","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:39.213Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"64ea071f-9748-4847-a81b-3f7db4382128","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:39.213Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"64ea071f-9748-4847-a81b-3f7db4382128","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:39.214Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"64ea071f-9748-4847-a81b-3f7db4382128"}
{"level":"info","ts":"2026-09-18T10:11:39.214Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"64ea071f-9748-4847-a81b-3f7db4382128","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:39.214Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"64ea071f-9748-4847-a81b-3f7db4382128","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:39.706Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"731a5b27-0a88-4c3c-97fa-af96a303fb1b"}
{"level":"info","ts":"2026-09-18T10:11:39.706Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"731a5b27-0a88-4c3c-97fa-af96a303fb1b"}
{"level":"info","ts":"2026-09-18T10:11:39.706Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"731a5b27-0a88-4c3c-97fa-af96a303fb1b"}
{"level":"info","ts":"2026-09-18T10:11:39.706Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"731a5b27-0a88-4c3c-97fa-af96a303fb1b","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:39.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"731a5b27-0a88-4c3c-97fa-af96a303fb1b","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:39.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"731a5b27-0a88-4c3c-97fa-af96a303fb1b","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:39.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"731a5b27-0a88-4c3c-97fa-af96a303fb1b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:39.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"731a5b27-0a88-4c3c-97fa-af96a303fb1b","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:39.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"731a5b27-0a88-4c3c-97fa-af96a303fb1b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:39.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"731a5b27-0a88-4c3c-97fa-af96a303fb1b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:39.706Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"731a5b27-0a88-4c3c-97fa-af96a303fb1b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:39.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"731a5b27-0a88-4c3c-97fa-af96a303fb1b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:39.706Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"731a5b27-0a88-4c3c-97fa-af96a303fb1b","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:11:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:11:39.706Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"731a5b27-0a88-4c3c-97fa-af96a303fb1b","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:11:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:11:39.706Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"731a5b27-0a88-4c3c-97fa-af96a303fb1b","name":"ttl-waiting-4wsk7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:11:39.721Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"2242dc91-924b-4311-a15e-5bafaa821d7a","rayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:39.721Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"3fc0d401-b1b8-4e8d-9f8f-1d7a14826d4c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:39.721Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"3fc0d401-b1b8-4e8d-9f8f-1d7a14826d4c","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:39.721Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"3fc0d401-b1b8-4e8d-9f8f-1d7a14826d4c","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:39.721Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"2242dc91-924b-4311-a15e-5bafaa821d7a","pod":"ttl-waiting-4wsk7-head-q4ncm","ip":"10.132.0.58"}
{"level":"info","ts":"2026-09-18T10:11:39.721Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"2242dc91-924b-4311-a15e-5bafaa821d7a","pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","ip":"10.133.0.50"}
{"level":"info","ts":"2026-09-18T10:11:39.721Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"3fc0d401-b1b8-4e8d-9f8f-1d7a14826d4c"}
{"level":"info","ts":"2026-09-18T10:11:39.721Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"3fc0d401-b1b8-4e8d-9f8f-1d7a14826d4c","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:39.721Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"3fc0d401-b1b8-4e8d-9f8f-1d7a14826d4c","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:39.721Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"3fc0d401-b1b8-4e8d-9f8f-1d7a14826d4c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:11:39.721Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"3a7e3b9d-e119-40a4-a9fa-f9dc5094ec03","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:39.735Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b7474fdd-6288-441c-a4bf-39306ea8e9c9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:39.735Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b7474fdd-6288-441c-a4bf-39306ea8e9c9","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:39.735Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b7474fdd-6288-441c-a4bf-39306ea8e9c9","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:39.735Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b7474fdd-6288-441c-a4bf-39306ea8e9c9"}
{"level":"info","ts":"2026-09-18T10:11:39.735Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b7474fdd-6288-441c-a4bf-39306ea8e9c9","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:39.735Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b7474fdd-6288-441c-a4bf-39306ea8e9c9","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:39.738Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"2242dc91-924b-4311-a15e-5bafaa821d7a","certificate":"ray-head-cert-ttl-waiting-4wsk7","dnsNames":["ttl-waiting-4wsk7-head-svc","localhost","ttl-waiting-4wsk7-head-svc.test-ns-ltx56.svc","ttl-waiting-4wsk7-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.58","10.133.0.50","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:11:39.739Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"3a7e3b9d-e119-40a4-a9fa-f9dc5094ec03","name":"ttl-waiting-4wsk7-head"}
{"level":"info","ts":"2026-09-18T10:11:39.753Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"2242dc91-924b-4311-a15e-5bafaa821d7a","certificate":"ray-worker-cert-ttl-waiting-4wsk7","dnsNames":["ttl-waiting-4wsk7-worker-svc","localhost","ttl-waiting-4wsk7-small-group","ttl-waiting-4wsk7-worker-svc.test-ns-ltx56.svc","ttl-waiting-4wsk7-worker-svc.test-ns-ltx56.svc.cluster.local","*.ttl-waiting-4wsk7-worker-svc.test-ns-ltx56.svc","*.ttl-waiting-4wsk7-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.132.0.58","10.133.0.50","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:11:39.753Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"2242dc91-924b-4311-a15e-5bafaa821d7a","rayCluster":"ttl-waiting-4wsk7","podIPs":["10.132.0.58","10.133.0.50"]}
{"level":"info","ts":"2026-09-18T10:11:39.754Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"3a7e3b9d-e119-40a4-a9fa-f9dc5094ec03","name":"ttl-waiting-4wsk7-workers"}
{"level":"info","ts":"2026-09-18T10:11:39.754Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"3a7e3b9d-e119-40a4-a9fa-f9dc5094ec03","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:41.721Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d088a559-64d0-41e9-810f-4121946ec321"}
{"level":"info","ts":"2026-09-18T10:11:41.721Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d088a559-64d0-41e9-810f-4121946ec321"}
{"level":"info","ts":"2026-09-18T10:11:41.721Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d088a559-64d0-41e9-810f-4121946ec321"}
{"level":"info","ts":"2026-09-18T10:11:41.721Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d088a559-64d0-41e9-810f-4121946ec321","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:41.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d088a559-64d0-41e9-810f-4121946ec321","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:41.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d088a559-64d0-41e9-810f-4121946ec321","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:41.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d088a559-64d0-41e9-810f-4121946ec321","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:41.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d088a559-64d0-41e9-810f-4121946ec321","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:41.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d088a559-64d0-41e9-810f-4121946ec321","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:41.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d088a559-64d0-41e9-810f-4121946ec321","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:41.721Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d088a559-64d0-41e9-810f-4121946ec321","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:41.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d088a559-64d0-41e9-810f-4121946ec321","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:41.721Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d088a559-64d0-41e9-810f-4121946ec321","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:11:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:11:41.721Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d088a559-64d0-41e9-810f-4121946ec321","podName":"ttl-waiting-4wsk7-head-q4ncm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:11:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:11:41.721Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d088a559-64d0-41e9-810f-4121946ec321","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:11:41.721Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d088a559-64d0-41e9-810f-4121946ec321","seconds":300}
{"level":"info","ts":"2026-09-18T10:11:42.215Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b69e2589-fccb-48ba-a5a3-54469eee3c9a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:42.215Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b69e2589-fccb-48ba-a5a3-54469eee3c9a","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:42.215Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b69e2589-fccb-48ba-a5a3-54469eee3c9a","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:42.215Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b69e2589-fccb-48ba-a5a3-54469eee3c9a"}
{"level":"info","ts":"2026-09-18T10:11:42.215Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b69e2589-fccb-48ba-a5a3-54469eee3c9a","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:42.215Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b69e2589-fccb-48ba-a5a3-54469eee3c9a","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:45.215Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"49d034c4-3ee6-45f5-883b-794bded7fef7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:45.215Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"49d034c4-3ee6-45f5-883b-794bded7fef7","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:45.215Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"49d034c4-3ee6-45f5-883b-794bded7fef7","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:45.215Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"49d034c4-3ee6-45f5-883b-794bded7fef7"}
{"level":"info","ts":"2026-09-18T10:11:45.215Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"49d034c4-3ee6-45f5-883b-794bded7fef7","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:45.215Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"49d034c4-3ee6-45f5-883b-794bded7fef7","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:45.585Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"27501c95-cd88-4f56-a435-0f0cfa5b8ef3","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:11:45.585Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"27501c95-cd88-4f56-a435-0f0cfa5b8ef3","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:11:45.585Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"27501c95-cd88-4f56-a435-0f0cfa5b8ef3","pod":"long-running-5qfl8-small-group-worker-tbqtj","ip":"10.132.0.47"}
{"level":"info","ts":"2026-09-18T10:11:45.585Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"27501c95-cd88-4f56-a435-0f0cfa5b8ef3","rayCluster":"long-running-5qfl8","podIPs":["10.133.0.42","10.132.0.47"]}
{"level":"info","ts":"2026-09-18T10:11:48.217Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a844ec84-fe57-4f2e-a7cd-abc4472394d8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:48.217Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a844ec84-fe57-4f2e-a7cd-abc4472394d8","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:48.217Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a844ec84-fe57-4f2e-a7cd-abc4472394d8","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:48.217Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a844ec84-fe57-4f2e-a7cd-abc4472394d8"}
{"level":"info","ts":"2026-09-18T10:11:48.217Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a844ec84-fe57-4f2e-a7cd-abc4472394d8","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:48.217Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a844ec84-fe57-4f2e-a7cd-abc4472394d8","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:50.733Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6d027815-6656-417d-9a9d-ea615206d940"}
{"level":"info","ts":"2026-09-18T10:11:50.733Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6d027815-6656-417d-9a9d-ea615206d940"}
{"level":"info","ts":"2026-09-18T10:11:50.733Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6d027815-6656-417d-9a9d-ea615206d940"}
{"level":"info","ts":"2026-09-18T10:11:50.733Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6d027815-6656-417d-9a9d-ea615206d940","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:50.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6d027815-6656-417d-9a9d-ea615206d940","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:50.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6d027815-6656-417d-9a9d-ea615206d940","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:50.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6d027815-6656-417d-9a9d-ea615206d940","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:50.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6d027815-6656-417d-9a9d-ea615206d940","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:50.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6d027815-6656-417d-9a9d-ea615206d940","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:50.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6d027815-6656-417d-9a9d-ea615206d940","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:50.734Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6d027815-6656-417d-9a9d-ea615206d940","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:50.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6d027815-6656-417d-9a9d-ea615206d940","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:50.734Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6d027815-6656-417d-9a9d-ea615206d940","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:50.734Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6d027815-6656-417d-9a9d-ea615206d940","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:50.734Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"6d027815-6656-417d-9a9d-ea615206d940","name":"ttl-waiting-4wsk7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:11:50.748Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"6cb5ff7e-6684-4416-9ac3-478b03dfb374","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:50.748Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"54e10ef7-f120-4c5a-ace6-01a20ddf0c0d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:50.748Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"54e10ef7-f120-4c5a-ace6-01a20ddf0c0d","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:50.748Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"54e10ef7-f120-4c5a-ace6-01a20ddf0c0d","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:50.749Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"54e10ef7-f120-4c5a-ace6-01a20ddf0c0d"}
{"level":"info","ts":"2026-09-18T10:11:50.749Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"54e10ef7-f120-4c5a-ace6-01a20ddf0c0d","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:50.749Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"54e10ef7-f120-4c5a-ace6-01a20ddf0c0d","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:50.749Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"54e10ef7-f120-4c5a-ace6-01a20ddf0c0d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:11:50.749Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"a8a55471-7ae6-468a-8766-0fb336de6ac2","rayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:50.749Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"a8a55471-7ae6-468a-8766-0fb336de6ac2","pod":"ttl-waiting-4wsk7-head-q4ncm","ip":"10.132.0.58"}
{"level":"info","ts":"2026-09-18T10:11:50.749Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"a8a55471-7ae6-468a-8766-0fb336de6ac2","pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","ip":"10.133.0.50"}
{"level":"info","ts":"2026-09-18T10:11:50.749Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"a8a55471-7ae6-468a-8766-0fb336de6ac2","rayCluster":"ttl-waiting-4wsk7","podIPs":["10.132.0.58","10.133.0.50"]}
{"level":"info","ts":"2026-09-18T10:11:50.769Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"cbfe54c6-3ed3-4ad8-b440-97eac3a98a8e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:50.769Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"cbfe54c6-3ed3-4ad8-b440-97eac3a98a8e","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:50.769Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"cbfe54c6-3ed3-4ad8-b440-97eac3a98a8e","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:50.770Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"cbfe54c6-3ed3-4ad8-b440-97eac3a98a8e"}
{"level":"info","ts":"2026-09-18T10:11:50.770Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"cbfe54c6-3ed3-4ad8-b440-97eac3a98a8e","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:50.770Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"cbfe54c6-3ed3-4ad8-b440-97eac3a98a8e","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:50.773Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"6cb5ff7e-6684-4416-9ac3-478b03dfb374","name":"ttl-waiting-4wsk7-head"}
{"level":"info","ts":"2026-09-18T10:11:50.789Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"6cb5ff7e-6684-4416-9ac3-478b03dfb374","name":"ttl-waiting-4wsk7-workers"}
{"level":"info","ts":"2026-09-18T10:11:50.789Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"6cb5ff7e-6684-4416-9ac3-478b03dfb374","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:51.217Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a1c8540c-8f33-4e44-90f5-9155a5abc353","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:51.217Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a1c8540c-8f33-4e44-90f5-9155a5abc353","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:51.217Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a1c8540c-8f33-4e44-90f5-9155a5abc353","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:51.217Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a1c8540c-8f33-4e44-90f5-9155a5abc353"}
{"level":"info","ts":"2026-09-18T10:11:51.217Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a1c8540c-8f33-4e44-90f5-9155a5abc353","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:51.217Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a1c8540c-8f33-4e44-90f5-9155a5abc353","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:52.749Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"cbd765f1-49f3-40c6-a310-8942e5d5ccc5"}
{"level":"info","ts":"2026-09-18T10:11:52.749Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"cbd765f1-49f3-40c6-a310-8942e5d5ccc5"}
{"level":"info","ts":"2026-09-18T10:11:52.749Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"cbd765f1-49f3-40c6-a310-8942e5d5ccc5"}
{"level":"info","ts":"2026-09-18T10:11:52.749Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"cbd765f1-49f3-40c6-a310-8942e5d5ccc5","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:11:52.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"cbd765f1-49f3-40c6-a310-8942e5d5ccc5","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:11:52.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"cbd765f1-49f3-40c6-a310-8942e5d5ccc5","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:52.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"cbd765f1-49f3-40c6-a310-8942e5d5ccc5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:11:52.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"cbd765f1-49f3-40c6-a310-8942e5d5ccc5","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:11:52.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"cbd765f1-49f3-40c6-a310-8942e5d5ccc5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:52.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"cbd765f1-49f3-40c6-a310-8942e5d5ccc5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:11:52.749Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"cbd765f1-49f3-40c6-a310-8942e5d5ccc5","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:11:52.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"cbd765f1-49f3-40c6-a310-8942e5d5ccc5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:11:52.749Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"cbd765f1-49f3-40c6-a310-8942e5d5ccc5","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:52.749Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"cbd765f1-49f3-40c6-a310-8942e5d5ccc5","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:11:52.749Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"cbd765f1-49f3-40c6-a310-8942e5d5ccc5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:11:52.749Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"cbd765f1-49f3-40c6-a310-8942e5d5ccc5","seconds":300}
{"level":"info","ts":"2026-09-18T10:11:54.219Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"341d4497-ddad-4f1b-90a0-bd1cd081a228","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:54.219Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"341d4497-ddad-4f1b-90a0-bd1cd081a228","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:54.219Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"341d4497-ddad-4f1b-90a0-bd1cd081a228","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:54.219Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"341d4497-ddad-4f1b-90a0-bd1cd081a228"}
{"level":"info","ts":"2026-09-18T10:11:54.219Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"341d4497-ddad-4f1b-90a0-bd1cd081a228","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:54.219Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"341d4497-ddad-4f1b-90a0-bd1cd081a228","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:11:57.219Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"60a571e6-1cd3-470c-918f-a7bcd125c457","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:11:57.219Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"60a571e6-1cd3-470c-918f-a7bcd125c457","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:11:57.219Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"60a571e6-1cd3-470c-918f-a7bcd125c457","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:11:57.220Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"60a571e6-1cd3-470c-918f-a7bcd125c457"}
{"level":"info","ts":"2026-09-18T10:11:57.220Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"60a571e6-1cd3-470c-918f-a7bcd125c457","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:11:57.220Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"60a571e6-1cd3-470c-918f-a7bcd125c457","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:00.220Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b5e482e6-3b07-47a5-bf45-75419a340f6f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:00.220Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b5e482e6-3b07-47a5-bf45-75419a340f6f","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:00.220Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b5e482e6-3b07-47a5-bf45-75419a340f6f","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:00.220Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b5e482e6-3b07-47a5-bf45-75419a340f6f"}
{"level":"info","ts":"2026-09-18T10:12:00.220Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b5e482e6-3b07-47a5-bf45-75419a340f6f","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:00.220Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b5e482e6-3b07-47a5-bf45-75419a340f6f","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:00.527Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"be55daad-48e9-41fd-8ab1-02062ba218a7","rayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:00.527Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"be55daad-48e9-41fd-8ab1-02062ba218a7","pod":"ttl-waiting-4wsk7-head-q4ncm","ip":"10.132.0.58"}
{"level":"info","ts":"2026-09-18T10:12:00.527Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"be55daad-48e9-41fd-8ab1-02062ba218a7","pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","ip":"10.133.0.50"}
{"level":"info","ts":"2026-09-18T10:12:00.527Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"be55daad-48e9-41fd-8ab1-02062ba218a7","rayCluster":"ttl-waiting-4wsk7","podIPs":["10.132.0.58","10.133.0.50"]}
{"level":"info","ts":"2026-09-18T10:12:03.221Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"225c2383-f187-455f-a3ac-1db489770544","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:03.221Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"225c2383-f187-455f-a3ac-1db489770544","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:03.221Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"225c2383-f187-455f-a3ac-1db489770544","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:03.222Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"225c2383-f187-455f-a3ac-1db489770544"}
{"level":"info","ts":"2026-09-18T10:12:03.222Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"225c2383-f187-455f-a3ac-1db489770544","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:03.222Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"225c2383-f187-455f-a3ac-1db489770544","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:06.223Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"45315fc3-0c9b-4881-8f4d-58c746a294b8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:06.223Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"45315fc3-0c9b-4881-8f4d-58c746a294b8","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:06.223Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"45315fc3-0c9b-4881-8f4d-58c746a294b8","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:06.223Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"45315fc3-0c9b-4881-8f4d-58c746a294b8"}
{"level":"info","ts":"2026-09-18T10:12:06.223Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"45315fc3-0c9b-4881-8f4d-58c746a294b8","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:06.223Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"45315fc3-0c9b-4881-8f4d-58c746a294b8","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:09.224Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9223cabb-d67b-4790-95c5-b32c08bda65c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:09.224Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9223cabb-d67b-4790-95c5-b32c08bda65c","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:09.224Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9223cabb-d67b-4790-95c5-b32c08bda65c","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:09.224Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9223cabb-d67b-4790-95c5-b32c08bda65c"}
{"level":"info","ts":"2026-09-18T10:12:09.224Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9223cabb-d67b-4790-95c5-b32c08bda65c","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:09.224Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9223cabb-d67b-4790-95c5-b32c08bda65c","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:12.225Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"5e8b7cdb-0f44-41fa-8917-81c1354da36b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:12.225Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"5e8b7cdb-0f44-41fa-8917-81c1354da36b","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:12.225Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"5e8b7cdb-0f44-41fa-8917-81c1354da36b","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:12.225Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"5e8b7cdb-0f44-41fa-8917-81c1354da36b"}
{"level":"info","ts":"2026-09-18T10:12:12.225Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"5e8b7cdb-0f44-41fa-8917-81c1354da36b","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:12.225Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"5e8b7cdb-0f44-41fa-8917-81c1354da36b","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:15.226Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"77267662-435e-48f7-9bf6-16b7e87a6bb1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:15.226Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"77267662-435e-48f7-9bf6-16b7e87a6bb1","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:15.226Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"77267662-435e-48f7-9bf6-16b7e87a6bb1","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:15.226Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"77267662-435e-48f7-9bf6-16b7e87a6bb1"}
{"level":"info","ts":"2026-09-18T10:12:15.226Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"77267662-435e-48f7-9bf6-16b7e87a6bb1","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:15.226Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"77267662-435e-48f7-9bf6-16b7e87a6bb1","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:18.227Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"25d56dfd-5bd3-4d96-8ae3-48b752f38b47","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:18.227Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"25d56dfd-5bd3-4d96-8ae3-48b752f38b47","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:18.227Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"25d56dfd-5bd3-4d96-8ae3-48b752f38b47","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:18.227Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"25d56dfd-5bd3-4d96-8ae3-48b752f38b47"}
{"level":"info","ts":"2026-09-18T10:12:18.227Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"25d56dfd-5bd3-4d96-8ae3-48b752f38b47","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:18.227Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"25d56dfd-5bd3-4d96-8ae3-48b752f38b47","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:21.228Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"491a3cc0-f994-49d4-a90c-7f5916dd39e4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:21.228Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"491a3cc0-f994-49d4-a90c-7f5916dd39e4","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:21.228Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"491a3cc0-f994-49d4-a90c-7f5916dd39e4","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:21.228Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"491a3cc0-f994-49d4-a90c-7f5916dd39e4"}
{"level":"info","ts":"2026-09-18T10:12:21.228Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"491a3cc0-f994-49d4-a90c-7f5916dd39e4","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:21.228Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"491a3cc0-f994-49d4-a90c-7f5916dd39e4","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:24.229Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b9fd6517-fe5e-472a-91fc-e8957e1e6e3a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:24.229Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b9fd6517-fe5e-472a-91fc-e8957e1e6e3a","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:24.229Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b9fd6517-fe5e-472a-91fc-e8957e1e6e3a","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:24.229Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b9fd6517-fe5e-472a-91fc-e8957e1e6e3a"}
{"level":"info","ts":"2026-09-18T10:12:24.229Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b9fd6517-fe5e-472a-91fc-e8957e1e6e3a","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:24.229Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b9fd6517-fe5e-472a-91fc-e8957e1e6e3a","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:27.230Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"48224621-731b-477a-9d17-3e6c8d242653","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:27.230Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"48224621-731b-477a-9d17-3e6c8d242653","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:27.230Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"48224621-731b-477a-9d17-3e6c8d242653","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:27.230Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"48224621-731b-477a-9d17-3e6c8d242653"}
{"level":"info","ts":"2026-09-18T10:12:27.230Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"48224621-731b-477a-9d17-3e6c8d242653","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:27.230Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"48224621-731b-477a-9d17-3e6c8d242653","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:30.231Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"7a154ea7-faf9-414a-acfb-103615b32819","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:30.231Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"7a154ea7-faf9-414a-acfb-103615b32819","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:30.231Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"7a154ea7-faf9-414a-acfb-103615b32819","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:30.231Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"7a154ea7-faf9-414a-acfb-103615b32819"}
{"level":"info","ts":"2026-09-18T10:12:30.231Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"7a154ea7-faf9-414a-acfb-103615b32819","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:30.231Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"7a154ea7-faf9-414a-acfb-103615b32819","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:33.232Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b4365117-3de9-4926-9aea-238643bedc8a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:33.232Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b4365117-3de9-4926-9aea-238643bedc8a","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:33.232Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b4365117-3de9-4926-9aea-238643bedc8a","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:33.232Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b4365117-3de9-4926-9aea-238643bedc8a"}
{"level":"info","ts":"2026-09-18T10:12:33.232Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b4365117-3de9-4926-9aea-238643bedc8a","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:33.232Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b4365117-3de9-4926-9aea-238643bedc8a","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:36.233Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"c85f90fa-4a6e-4edd-b7ec-a91a353e4005","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:36.233Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"c85f90fa-4a6e-4edd-b7ec-a91a353e4005","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:36.233Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"c85f90fa-4a6e-4edd-b7ec-a91a353e4005","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:36.233Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"c85f90fa-4a6e-4edd-b7ec-a91a353e4005"}
{"level":"info","ts":"2026-09-18T10:12:36.233Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"c85f90fa-4a6e-4edd-b7ec-a91a353e4005","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:36.233Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"c85f90fa-4a6e-4edd-b7ec-a91a353e4005","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:39.066Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d32454dc-e673-4634-84ab-ba8d7fdaf957"}
{"level":"info","ts":"2026-09-18T10:12:39.066Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d32454dc-e673-4634-84ab-ba8d7fdaf957"}
{"level":"info","ts":"2026-09-18T10:12:39.066Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d32454dc-e673-4634-84ab-ba8d7fdaf957"}
{"level":"info","ts":"2026-09-18T10:12:39.066Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d32454dc-e673-4634-84ab-ba8d7fdaf957","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:12:39.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d32454dc-e673-4634-84ab-ba8d7fdaf957","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:12:39.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d32454dc-e673-4634-84ab-ba8d7fdaf957","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:12:39.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d32454dc-e673-4634-84ab-ba8d7fdaf957","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:12:39.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d32454dc-e673-4634-84ab-ba8d7fdaf957","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:12:39.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d32454dc-e673-4634-84ab-ba8d7fdaf957","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:12:39.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d32454dc-e673-4634-84ab-ba8d7fdaf957","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:12:39.067Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d32454dc-e673-4634-84ab-ba8d7fdaf957","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:12:39.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d32454dc-e673-4634-84ab-ba8d7fdaf957","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:12:39.067Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d32454dc-e673-4634-84ab-ba8d7fdaf957","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:12:39.067Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d32454dc-e673-4634-84ab-ba8d7fdaf957","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:12:39.067Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d32454dc-e673-4634-84ab-ba8d7fdaf957","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:12:39.067Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"d32454dc-e673-4634-84ab-ba8d7fdaf957","seconds":300}
{"level":"info","ts":"2026-09-18T10:12:39.234Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"8dba3d11-61ca-4983-8c4e-c5525a0a21b2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:39.234Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"8dba3d11-61ca-4983-8c4e-c5525a0a21b2","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:39.234Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"8dba3d11-61ca-4983-8c4e-c5525a0a21b2","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:39.234Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"8dba3d11-61ca-4983-8c4e-c5525a0a21b2"}
{"level":"info","ts":"2026-09-18T10:12:39.234Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"8dba3d11-61ca-4983-8c4e-c5525a0a21b2","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:39.234Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"8dba3d11-61ca-4983-8c4e-c5525a0a21b2","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:40.082Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b714ad66-dc38-4d17-b474-f5ded874aa9f"}
{"level":"info","ts":"2026-09-18T10:12:40.082Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b714ad66-dc38-4d17-b474-f5ded874aa9f"}
{"level":"info","ts":"2026-09-18T10:12:40.082Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b714ad66-dc38-4d17-b474-f5ded874aa9f"}
{"level":"info","ts":"2026-09-18T10:12:40.082Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b714ad66-dc38-4d17-b474-f5ded874aa9f","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:12:40.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b714ad66-dc38-4d17-b474-f5ded874aa9f","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:12:40.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b714ad66-dc38-4d17-b474-f5ded874aa9f","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:12:40.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b714ad66-dc38-4d17-b474-f5ded874aa9f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:12:40.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b714ad66-dc38-4d17-b474-f5ded874aa9f","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:12:40.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b714ad66-dc38-4d17-b474-f5ded874aa9f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:12:40.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b714ad66-dc38-4d17-b474-f5ded874aa9f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:12:40.082Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b714ad66-dc38-4d17-b474-f5ded874aa9f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:12:40.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b714ad66-dc38-4d17-b474-f5ded874aa9f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:12:40.082Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b714ad66-dc38-4d17-b474-f5ded874aa9f","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:12:40.082Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b714ad66-dc38-4d17-b474-f5ded874aa9f","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:12:40.082Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b714ad66-dc38-4d17-b474-f5ded874aa9f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:12:40.082Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b714ad66-dc38-4d17-b474-f5ded874aa9f","seconds":300}
{"level":"info","ts":"2026-09-18T10:12:42.234Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"801be9e4-a106-4c10-932b-bd9791c338bc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:42.234Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"801be9e4-a106-4c10-932b-bd9791c338bc","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:42.234Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"801be9e4-a106-4c10-932b-bd9791c338bc","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:42.234Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"801be9e4-a106-4c10-932b-bd9791c338bc"}
{"level":"info","ts":"2026-09-18T10:12:42.234Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"801be9e4-a106-4c10-932b-bd9791c338bc","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:42.234Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"801be9e4-a106-4c10-932b-bd9791c338bc","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:45.235Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a6df5369-48f8-4d74-a6b7-a6190df53ce7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:45.235Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a6df5369-48f8-4d74-a6b7-a6190df53ce7","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:45.235Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a6df5369-48f8-4d74-a6b7-a6190df53ce7","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:45.235Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a6df5369-48f8-4d74-a6b7-a6190df53ce7"}
{"level":"info","ts":"2026-09-18T10:12:45.235Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a6df5369-48f8-4d74-a6b7-a6190df53ce7","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:45.235Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"a6df5369-48f8-4d74-a6b7-a6190df53ce7","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:45.585Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a0904cda-3ab5-4b08-80a2-25838095de3a","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:12:45.585Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a0904cda-3ab5-4b08-80a2-25838095de3a","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:12:45.585Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a0904cda-3ab5-4b08-80a2-25838095de3a","pod":"long-running-5qfl8-small-group-worker-tbqtj","ip":"10.132.0.47"}
{"level":"info","ts":"2026-09-18T10:12:45.585Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a0904cda-3ab5-4b08-80a2-25838095de3a","rayCluster":"long-running-5qfl8","podIPs":["10.133.0.42","10.132.0.47"]}
{"level":"info","ts":"2026-09-18T10:12:48.236Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b84848bb-1972-4234-88b5-ba54dcc19900","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:48.236Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b84848bb-1972-4234-88b5-ba54dcc19900","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:48.236Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b84848bb-1972-4234-88b5-ba54dcc19900","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:48.236Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b84848bb-1972-4234-88b5-ba54dcc19900"}
{"level":"info","ts":"2026-09-18T10:12:48.236Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b84848bb-1972-4234-88b5-ba54dcc19900","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:48.236Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"b84848bb-1972-4234-88b5-ba54dcc19900","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:49.138Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b8e42a0b-824f-4c14-b7b6-57ce91f9a431"}
{"level":"info","ts":"2026-09-18T10:12:49.139Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b8e42a0b-824f-4c14-b7b6-57ce91f9a431"}
{"level":"info","ts":"2026-09-18T10:12:49.139Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b8e42a0b-824f-4c14-b7b6-57ce91f9a431"}
{"level":"info","ts":"2026-09-18T10:12:49.139Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b8e42a0b-824f-4c14-b7b6-57ce91f9a431","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:12:49.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b8e42a0b-824f-4c14-b7b6-57ce91f9a431","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:12:49.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b8e42a0b-824f-4c14-b7b6-57ce91f9a431","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:12:49.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b8e42a0b-824f-4c14-b7b6-57ce91f9a431","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:12:49.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b8e42a0b-824f-4c14-b7b6-57ce91f9a431","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:12:49.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b8e42a0b-824f-4c14-b7b6-57ce91f9a431","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:12:49.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b8e42a0b-824f-4c14-b7b6-57ce91f9a431","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:12:49.139Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b8e42a0b-824f-4c14-b7b6-57ce91f9a431","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:12:49.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b8e42a0b-824f-4c14-b7b6-57ce91f9a431","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:12:49.139Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b8e42a0b-824f-4c14-b7b6-57ce91f9a431","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:12:49.139Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b8e42a0b-824f-4c14-b7b6-57ce91f9a431","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:12:49.139Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b8e42a0b-824f-4c14-b7b6-57ce91f9a431","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:12:49.139Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"b8e42a0b-824f-4c14-b7b6-57ce91f9a431","seconds":300}
{"level":"info","ts":"2026-09-18T10:12:50.136Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"712f63b9-5588-4dd1-a580-ec436d2ea15d"}
{"level":"info","ts":"2026-09-18T10:12:50.136Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"712f63b9-5588-4dd1-a580-ec436d2ea15d"}
{"level":"info","ts":"2026-09-18T10:12:50.136Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"712f63b9-5588-4dd1-a580-ec436d2ea15d"}
{"level":"info","ts":"2026-09-18T10:12:50.136Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"712f63b9-5588-4dd1-a580-ec436d2ea15d","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:12:50.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"712f63b9-5588-4dd1-a580-ec436d2ea15d","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:12:50.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"712f63b9-5588-4dd1-a580-ec436d2ea15d","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:12:50.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"712f63b9-5588-4dd1-a580-ec436d2ea15d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:12:50.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"712f63b9-5588-4dd1-a580-ec436d2ea15d","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:12:50.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"712f63b9-5588-4dd1-a580-ec436d2ea15d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:12:50.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"712f63b9-5588-4dd1-a580-ec436d2ea15d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:12:50.136Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"712f63b9-5588-4dd1-a580-ec436d2ea15d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:12:50.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"712f63b9-5588-4dd1-a580-ec436d2ea15d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:12:50.136Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"712f63b9-5588-4dd1-a580-ec436d2ea15d","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:11:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:12:50.136Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"712f63b9-5588-4dd1-a580-ec436d2ea15d","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:11:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:12:50.136Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"712f63b9-5588-4dd1-a580-ec436d2ea15d","name":"ttl-waiting-4wsk7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:12:50.158Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"f27cf80f-4c1a-4dd7-afe3-8fc11a57190d","rayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:50.158Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"47f4ae8e-b2f6-408d-ab68-71f11cc4ec05","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:50.158Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"f27cf80f-4c1a-4dd7-afe3-8fc11a57190d","pod":"ttl-waiting-4wsk7-head-q4ncm","ip":"10.132.0.58"}
{"level":"info","ts":"2026-09-18T10:12:50.158Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"f27cf80f-4c1a-4dd7-afe3-8fc11a57190d","pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","ip":"10.133.0.50"}
{"level":"info","ts":"2026-09-18T10:12:50.158Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"f27cf80f-4c1a-4dd7-afe3-8fc11a57190d","rayCluster":"ttl-waiting-4wsk7","podIPs":["10.132.0.58","10.133.0.50"]}
{"level":"info","ts":"2026-09-18T10:12:50.158Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"109d3d24-474a-4dad-9dc6-39587af6b620","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:50.158Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"109d3d24-474a-4dad-9dc6-39587af6b620","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:50.158Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"109d3d24-474a-4dad-9dc6-39587af6b620","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:50.158Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"109d3d24-474a-4dad-9dc6-39587af6b620"}
{"level":"info","ts":"2026-09-18T10:12:50.158Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"109d3d24-474a-4dad-9dc6-39587af6b620","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:50.159Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"109d3d24-474a-4dad-9dc6-39587af6b620","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:11:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:50.159Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"109d3d24-474a-4dad-9dc6-39587af6b620","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-09-18T10:12:50.176Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e83aa7b4-5051-4ac1-ba25-8bf545c8bc94","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:50.176Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e83aa7b4-5051-4ac1-ba25-8bf545c8bc94","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:50.176Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e83aa7b4-5051-4ac1-ba25-8bf545c8bc94","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:50.176Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e83aa7b4-5051-4ac1-ba25-8bf545c8bc94"}
{"level":"info","ts":"2026-09-18T10:12:50.176Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e83aa7b4-5051-4ac1-ba25-8bf545c8bc94","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:50.176Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e83aa7b4-5051-4ac1-ba25-8bf545c8bc94","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:50.186Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"47f4ae8e-b2f6-408d-ab68-71f11cc4ec05","name":"ttl-waiting-4wsk7-head"}
{"level":"info","ts":"2026-09-18T10:12:50.200Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"47f4ae8e-b2f6-408d-ab68-71f11cc4ec05","name":"ttl-waiting-4wsk7-workers"}
{"level":"info","ts":"2026-09-18T10:12:50.200Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"47f4ae8e-b2f6-408d-ab68-71f11cc4ec05","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:51.237Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"d4f7ed3b-fe4d-4b0d-ae36-911ad954be66","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:51.237Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"d4f7ed3b-fe4d-4b0d-ae36-911ad954be66","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:51.237Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"d4f7ed3b-fe4d-4b0d-ae36-911ad954be66","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:51.237Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"d4f7ed3b-fe4d-4b0d-ae36-911ad954be66"}
{"level":"info","ts":"2026-09-18T10:12:51.237Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"d4f7ed3b-fe4d-4b0d-ae36-911ad954be66","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:51.237Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"d4f7ed3b-fe4d-4b0d-ae36-911ad954be66","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:52.158Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"8368bd78-fae8-4c3c-914b-f29f38e971e3"}
{"level":"info","ts":"2026-09-18T10:12:52.158Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"8368bd78-fae8-4c3c-914b-f29f38e971e3"}
{"level":"info","ts":"2026-09-18T10:12:52.158Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"8368bd78-fae8-4c3c-914b-f29f38e971e3"}
{"level":"info","ts":"2026-09-18T10:12:52.158Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"8368bd78-fae8-4c3c-914b-f29f38e971e3","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:12:52.158Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"8368bd78-fae8-4c3c-914b-f29f38e971e3","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:12:52.158Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"8368bd78-fae8-4c3c-914b-f29f38e971e3","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:12:52.158Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"8368bd78-fae8-4c3c-914b-f29f38e971e3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:12:52.158Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"8368bd78-fae8-4c3c-914b-f29f38e971e3","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:12:52.158Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"8368bd78-fae8-4c3c-914b-f29f38e971e3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:12:52.158Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"8368bd78-fae8-4c3c-914b-f29f38e971e3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:12:52.158Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"8368bd78-fae8-4c3c-914b-f29f38e971e3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:12:52.158Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"8368bd78-fae8-4c3c-914b-f29f38e971e3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:12:52.158Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"8368bd78-fae8-4c3c-914b-f29f38e971e3","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:11:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:12:52.158Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"8368bd78-fae8-4c3c-914b-f29f38e971e3","podName":"ttl-waiting-4wsk7-small-group-worker-vfzdt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:11:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:12:52.159Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"8368bd78-fae8-4c3c-914b-f29f38e971e3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:12:52.159Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"8368bd78-fae8-4c3c-914b-f29f38e971e3","seconds":300}
{"level":"info","ts":"2026-09-18T10:12:54.238Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e8ed2936-3dcd-48c7-819d-e8d43a827ae7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:54.238Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e8ed2936-3dcd-48c7-819d-e8d43a827ae7","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:54.238Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e8ed2936-3dcd-48c7-819d-e8d43a827ae7","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:54.238Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e8ed2936-3dcd-48c7-819d-e8d43a827ae7"}
{"level":"info","ts":"2026-09-18T10:12:54.238Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e8ed2936-3dcd-48c7-819d-e8d43a827ae7","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:54.238Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"e8ed2936-3dcd-48c7-819d-e8d43a827ae7","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:12:57.239Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"f9fe8643-c4ab-4f65-a42f-184dd0ed0322","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:12:57.239Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"f9fe8643-c4ab-4f65-a42f-184dd0ed0322","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:12:57.239Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"f9fe8643-c4ab-4f65-a42f-184dd0ed0322","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:12:57.239Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"f9fe8643-c4ab-4f65-a42f-184dd0ed0322"}
{"level":"info","ts":"2026-09-18T10:12:57.239Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"f9fe8643-c4ab-4f65-a42f-184dd0ed0322","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:12:57.239Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"f9fe8643-c4ab-4f65-a42f-184dd0ed0322","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:13:00.240Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9d001eb2-11df-4a83-938a-b2d83f3cc5e3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:13:00.240Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9d001eb2-11df-4a83-938a-b2d83f3cc5e3","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:00.240Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9d001eb2-11df-4a83-938a-b2d83f3cc5e3","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:13:00.240Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9d001eb2-11df-4a83-938a-b2d83f3cc5e3"}
{"level":"info","ts":"2026-09-18T10:13:00.240Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9d001eb2-11df-4a83-938a-b2d83f3cc5e3","RayCluster":"ttl-waiting-4wsk7","State":""}
{"level":"info","ts":"2026-09-18T10:13:00.240Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9d001eb2-11df-4a83-938a-b2d83f3cc5e3","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:13:00.528Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"ee80bd1b-f9e1-43b4-916d-72a9845a3351","rayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:00.528Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"ee80bd1b-f9e1-43b4-916d-72a9845a3351","pod":"ttl-waiting-4wsk7-head-q4ncm","ip":"10.132.0.58"}
{"level":"info","ts":"2026-09-18T10:13:00.528Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"ee80bd1b-f9e1-43b4-916d-72a9845a3351","pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","ip":"10.133.0.50"}
{"level":"info","ts":"2026-09-18T10:13:00.528Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"ee80bd1b-f9e1-43b4-916d-72a9845a3351","rayCluster":"ttl-waiting-4wsk7","podIPs":["10.132.0.58","10.133.0.50"]}
{"level":"info","ts":"2026-09-18T10:13:01.146Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ad60cbcf-0e0e-4303-a03f-c804c28766c8"}
{"level":"info","ts":"2026-09-18T10:13:01.146Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ad60cbcf-0e0e-4303-a03f-c804c28766c8"}
{"level":"info","ts":"2026-09-18T10:13:01.147Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ad60cbcf-0e0e-4303-a03f-c804c28766c8"}
{"level":"info","ts":"2026-09-18T10:13:01.147Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ad60cbcf-0e0e-4303-a03f-c804c28766c8","1 head service found":"ttl-waiting-4wsk7-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:01.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ad60cbcf-0e0e-4303-a03f-c804c28766c8","Found 1 head Pod":"ttl-waiting-4wsk7-head-q4ncm","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:01.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ad60cbcf-0e0e-4303-a03f-c804c28766c8","head Pod":"ttl-waiting-4wsk7-head-q4ncm","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-4wsk7-head-q4ncm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:01.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ad60cbcf-0e0e-4303-a03f-c804c28766c8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:01.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ad60cbcf-0e0e-4303-a03f-c804c28766c8","worker Pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-4wsk7-small-group-worker-vfzdt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:01.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ad60cbcf-0e0e-4303-a03f-c804c28766c8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:01.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ad60cbcf-0e0e-4303-a03f-c804c28766c8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:01.147Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ad60cbcf-0e0e-4303-a03f-c804c28766c8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:13:01.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ad60cbcf-0e0e-4303-a03f-c804c28766c8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:01.147Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"ad60cbcf-0e0e-4303-a03f-c804c28766c8","name":"ttl-waiting-4wsk7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:01Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:01Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:01Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:13:01.164Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"080c9012-1776-465f-99eb-00f10953374a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:13:01.164Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"080c9012-1776-465f-99eb-00f10953374a","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:01.164Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"080c9012-1776-465f-99eb-00f10953374a","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:13:01.164Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"080c9012-1776-465f-99eb-00f10953374a"}
{"level":"info","ts":"2026-09-18T10:13:01.164Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"230e5fdf-ad6d-4507-9c90-36e2d9ae4a58","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:01.164Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"080c9012-1776-465f-99eb-00f10953374a","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","jobDeploymentStatus":"Initializing","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","dashboardURL":"ttl-waiting-4wsk7-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Waiting","startTime":"2026-09-18T10:11:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:13:01.164Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"080c9012-1776-465f-99eb-00f10953374a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Waiting"}
{"level":"info","ts":"2026-09-18T10:13:01.165Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"18d4b2e4-a8b0-4224-b576-0ab29be86c83","rayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:01.165Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"18d4b2e4-a8b0-4224-b576-0ab29be86c83","pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","ip":"10.133.0.50"}
{"level":"info","ts":"2026-09-18T10:13:01.165Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"18d4b2e4-a8b0-4224-b576-0ab29be86c83","pod":"ttl-waiting-4wsk7-head-q4ncm","ip":"10.132.0.58"}
{"level":"info","ts":"2026-09-18T10:13:01.165Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"18d4b2e4-a8b0-4224-b576-0ab29be86c83","rayCluster":"ttl-waiting-4wsk7","podIPs":["10.133.0.50","10.132.0.58"]}
{"level":"info","ts":"2026-09-18T10:13:01.178Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"4ac91fbe-e973-4ba2-af9f-f7159ad1f7bd","JobStatus":"","JobDeploymentStatus":"Waiting","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:13:01.183Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"230e5fdf-ad6d-4507-9c90-36e2d9ae4a58","name":"ttl-waiting-4wsk7-head"}
{"level":"info","ts":"2026-09-18T10:13:01.195Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"230e5fdf-ad6d-4507-9c90-36e2d9ae4a58","name":"ttl-waiting-4wsk7-workers"}
{"level":"info","ts":"2026-09-18T10:13:01.195Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"230e5fdf-ad6d-4507-9c90-36e2d9ae4a58","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:01.895Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"973757b1-52d8-428b-91bf-1fd54e6094bb","JobStatus":"","JobDeploymentStatus":"Waiting","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:13:01.895Z","logger":"controllers.RayJob","msg":"The RayJob has passed the preRunningDeadlineSeconds. Transition the status to `Failed`.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"973757b1-52d8-428b-91bf-1fd54e6094bb","StartTime":"2026-09-18 10:07:56 +0000 UTC","PreRunningDeadlineSeconds":300}
{"level":"info","ts":"2026-09-18T10:13:01.895Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"973757b1-52d8-428b-91bf-1fd54e6094bb","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","dashboardURL":"ttl-waiting-4wsk7-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Waiting","startTime":"2026-09-18T10:07:56Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-4wsk7","dashboardURL":"ttl-waiting-4wsk7-head-svc.test-ns-ltx56.svc.cluster.local:8265","jobDeploymentStatus":"Failed","reason":"PreRunningDeadlineExceeded","message":"The RayJob has passed the preRunningDeadlineSeconds. StartTime: 2026-09-18 10:07:56 +0000 UTC. PreRunningDeadlineSeconds: 300","startTime":"2026-09-18T10:07:56Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:12:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.58","serviceIP":"10.132.0.58","podName":"ttl-waiting-4wsk7-head-q4ncm","serviceName":"ttl-waiting-4wsk7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:11:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:11:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:11:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-09-18T10:13:01.895Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"973757b1-52d8-428b-91bf-1fd54e6094bb","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Waiting","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-09-18T10:13:01.912Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"fee94474-8087-471a-abb2-b842a24e0bd1","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:13:01.912Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"fee94474-8087-471a-abb2-b842a24e0bd1","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:13:01 +0000 UTC","Now":"2026-09-18T10:13:01.912Z","ShutdownTime":"2026-09-18T10:13:01.000Z"}
{"level":"info","ts":"2026-09-18T10:13:01.925Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"c0eb26ec-cb86-43c2-ac22-18ca65c5ebde"}
{"level":"info","ts":"2026-09-18T10:13:01.925Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"0df4e812-eb4f-4c2c-bef6-e0faa55190b6","namespacedName":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"0df4e812-eb4f-4c2c-bef6-e0faa55190b6","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"0df4e812-eb4f-4c2c-bef6-e0faa55190b6","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"reconcileID":"283ea88e-6658-45ac-8420-f1b483c52e4a"}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"0df4e812-eb4f-4c2c-bef6-e0faa55190b6","namespace":"test-ns-ltx56","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"0df4e812-eb4f-4c2c-bef6-e0faa55190b6","name":"kuberay-gateway-access","namespace":"test-ns-ltx56","remainingClusters":1}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"c6d5939c-b5e5-4112-a8fa-d8640196d62f","rayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"c6d5939c-b5e5-4112-a8fa-d8640196d62f","pod":"ttl-waiting-4wsk7-head-q4ncm","ip":"10.132.0.58"}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"c6d5939c-b5e5-4112-a8fa-d8640196d62f","pod":"ttl-waiting-4wsk7-small-group-worker-vfzdt","ip":"10.133.0.50"}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"c6d5939c-b5e5-4112-a8fa-d8640196d62f","rayCluster":"ttl-waiting-4wsk7","podIPs":["10.132.0.58","10.133.0.50"]}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"fee94474-8087-471a-abb2-b842a24e0bd1","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"fee94474-8087-471a-abb2-b842a24e0bd1","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"fee94474-8087-471a-abb2-b842a24e0bd1","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"98008152-712d-4714-b058-f396d534f320","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"98008152-712d-4714-b058-f396d534f320","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:13:01 +0000 UTC","Now":"2026-09-18T10:13:01.926Z","ShutdownTime":"2026-09-18T10:13:01.000Z"}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"98008152-712d-4714-b058-f396d534f320","RayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"98008152-712d-4714-b058-f396d534f320","isClusterDeleted":false}
{"level":"info","ts":"2026-09-18T10:13:01.926Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"98008152-712d-4714-b058-f396d534f320","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:01.934Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"0df4e812-eb4f-4c2c-bef6-e0faa55190b6","name":"test-ns-ltx56-ttl-waiting-4wsk7","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T10:13:01.939Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"0df4e812-eb4f-4c2c-bef6-e0faa55190b6","configMap":"kube-rbac-proxy-config-ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:01.945Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"0df4e812-eb4f-4c2c-bef6-e0faa55190b6","serviceAccount":"ttl-waiting-4wsk7-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:13:01.973Z","logger":"raycluster-resource","msg":"default","name":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:01.973Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:13:01.986Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"0df4e812-eb4f-4c2c-bef6-e0faa55190b6","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:01.986Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"6fcf05f5-e529-4ab8-a244-88ad13116aca","namespacedName":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:13:01.986Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"6fcf05f5-e529-4ab8-a244-88ad13116aca","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:01.986Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"6fcf05f5-e529-4ab8-a244-88ad13116aca","cluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:01.986Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"6fcf05f5-e529-4ab8-a244-88ad13116aca","namespace":"test-ns-ltx56","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-09-18T10:13:01.986Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"6fcf05f5-e529-4ab8-a244-88ad13116aca","name":"kuberay-gateway-access","namespace":"test-ns-ltx56","remainingClusters":1}
{"level":"info","ts":"2026-09-18T10:13:01.987Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"3cd4fdcb-4cfa-4b96-bef7-3e716b0fe800"}
{"level":"info","ts":"2026-09-18T10:13:01.987Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"00accffe-8ba8-4024-b477-01540b5af033","namespace":"test-ns-ltx56","clusterName":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:01.988Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"4e42895b-5308-4dd4-a2f5-278f55f7e1d3","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:13:01.988Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"4e42895b-5308-4dd4-a2f5-278f55f7e1d3","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:13:01 +0000 UTC","Now":"2026-09-18T10:13:01.988Z","ShutdownTime":"2026-09-18T10:13:01.000Z"}
{"level":"info","ts":"2026-09-18T10:13:01.988Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"4e42895b-5308-4dd4-a2f5-278f55f7e1d3","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:13:01.988Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"4e42895b-5308-4dd4-a2f5-278f55f7e1d3","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T10:13:01.988Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"4e42895b-5308-4dd4-a2f5-278f55f7e1d3","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"ttl-waiting-4wsk7"}
{"level":"error","ts":"2026-09-18T10:13:01.992Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"6fcf05f5-e529-4ab8-a244-88ad13116aca","error":"rayclusters.ray.io \"ttl-waiting-4wsk7\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:196\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:137\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:01.992Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"6fcf05f5-e529-4ab8-a244-88ad13116aca"}
{"level":"error","ts":"2026-09-18T10:13:01.992Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"6fcf05f5-e529-4ab8-a244-88ad13116aca","error":"rayclusters.ray.io \"ttl-waiting-4wsk7\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:01.992Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"aba10d9f-8edc-4748-89ff-e08f2e2e83dc","namespacedName":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:13:01.992Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"aba10d9f-8edc-4748-89ff-e08f2e2e83dc"}
{"level":"info","ts":"2026-09-18T10:13:01.992Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"aba10d9f-8edc-4748-89ff-e08f2e2e83dc","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:13:01.992Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"aba10d9f-8edc-4748-89ff-e08f2e2e83dc","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:13:01.997Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"1f1369bf-f41d-4447-92cc-c15fa79e74c6","namespacedName":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:13:01.997Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"1f1369bf-f41d-4447-92cc-c15fa79e74c6"}
{"level":"info","ts":"2026-09-18T10:13:01.997Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"1f1369bf-f41d-4447-92cc-c15fa79e74c6","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:13:01.997Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"1f1369bf-f41d-4447-92cc-c15fa79e74c6","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:13:02.009Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"f91b0944-ff9a-4c46-bff0-6818a6cc9759"}
{"level":"info","ts":"2026-09-18T10:13:02.009Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"75f9150d-21ca-4b85-8470-2fbb4b08b3aa"}
{"level":"info","ts":"2026-09-18T10:13:02.031Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"a0a8e02e-1cea-4ae4-9379-242d06d30fcb","namespacedName":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:13:02.031Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"a0a8e02e-1cea-4ae4-9379-242d06d30fcb"}
{"level":"info","ts":"2026-09-18T10:13:02.031Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"a0a8e02e-1cea-4ae4-9379-242d06d30fcb","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:13:02.031Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"a0a8e02e-1cea-4ae4-9379-242d06d30fcb","namespace":"test-ns-ltx56","authClusters":1}
{"level":"info","ts":"2026-09-18T10:13:02.056Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"00accffe-8ba8-4024-b477-01540b5af033","namespace":"test-ns-ltx56","clusterName":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:03.241Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"08a6cf29-58ad-4946-8303-86de4ff023c8","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-09-18T10:13:03.241Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"08a6cf29-58ad-4946-8303-86de4ff023c8","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-09-18 10:13:01 +0000 UTC","Now":"2026-09-18T10:13:03.241Z","ShutdownTime":"2026-09-18T10:13:01.000Z"}
{"level":"info","ts":"2026-09-18T10:13:03.241Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"08a6cf29-58ad-4946-8303-86de4ff023c8","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:13:03.241Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"08a6cf29-58ad-4946-8303-86de4ff023c8","isClusterDeleted":true}
{"level":"info","ts":"2026-09-18T10:13:03.241Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"08a6cf29-58ad-4946-8303-86de4ff023c8","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:13:04.925Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:04.925Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-pmtv8"}
{"level":"info","ts":"2026-09-18T10:13:04.938Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"8494c17a-b472-4ef3-a149-cffe6517c9d0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:04.938Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e1e28951-b6bc-4942-a82f-0041ac8959bb","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pmtv8"}}
{"level":"info","ts":"2026-09-18T10:13:04.938Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e1e28951-b6bc-4942-a82f-0041ac8959bb","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:04.938Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e1e28951-b6bc-4942-a82f-0041ac8959bb","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:04.938Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e1e28951-b6bc-4942-a82f-0041ac8959bb","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:04.938Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1e401f55-0071-416b-bdc6-11384d0c6cb7","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:04.938Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1e401f55-0071-416b-bdc6-11384d0c6cb7","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:04.945Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:04.945Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-pmtv8"}
{"level":"info","ts":"2026-09-18T10:13:04.946Z","logger":"controllers.RayCluster","msg":"Created service account for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4b3c500f-4f2f-4505-a545-c49f1e17054e","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:04.953Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1e401f55-0071-416b-bdc6-11384d0c6cb7","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:04.953Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1e401f55-0071-416b-bdc6-11384d0c6cb7","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:04.960Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"8494c17a-b472-4ef3-a149-cffe6517c9d0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:04.960Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e4a42c70-8fdf-4695-8b55-69648e1f8765","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:04.970Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1e401f55-0071-416b-bdc6-11384d0c6cb7","rayCluster":"ray-cluster","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T10:13:04.970Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1e401f55-0071-416b-bdc6-11384d0c6cb7","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:04.971Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e1e28951-b6bc-4942-a82f-0041ac8959bb","name":"ray-cluster-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:13:04.981Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1e401f55-0071-416b-bdc6-11384d0c6cb7","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:04.986Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e4a42c70-8fdf-4695-8b55-69648e1f8765","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:13:05.067Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1e401f55-0071-416b-bdc6-11384d0c6cb7","rayCluster":"ray-cluster","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:13:05.165Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e1e28951-b6bc-4942-a82f-0041ac8959bb","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.166Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e4a42c70-8fdf-4695-8b55-69648e1f8765","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:13:05.166Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e4a42c70-8fdf-4695-8b55-69648e1f8765","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.166Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"fb699be6-ab0c-4929-b57d-64f4ff7c819d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.166Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1e401f55-0071-416b-bdc6-11384d0c6cb7","rayCluster":"ray-cluster","podIPs":[]}
{"level":"error","ts":"2026-09-18T10:13:05.166Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1e401f55-0071-416b-bdc6-11384d0c6cb7","error":"Certificate.cert-manager.io \"ray-worker-cert-ray-cluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.166Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1e401f55-0071-416b-bdc6-11384d0c6cb7"}
{"level":"error","ts":"2026-09-18T10:13:05.166Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1e401f55-0071-416b-bdc6-11384d0c6cb7","error":"Certificate.cert-manager.io \"ray-worker-cert-ray-cluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.166Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d80bba21-fa44-4e56-a796-216d9270d936","rayCluster":"ray-cluster"}
{"level":"error","ts":"2026-09-18T10:13:05.173Z","logger":"authentication-controller","msg":"Failed to update AuthenticationReady condition","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e1e28951-b6bc-4942-a82f-0041ac8959bb","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleOIDCConfiguration\n\t/workspace/controllers/ray/authentication_controller.go:281\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:156\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.173Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e1e28951-b6bc-4942-a82f-0041ac8959bb","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.173Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"65389991-1854-4e00-a3fe-208529240aad","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pmtv8"}}
{"level":"info","ts":"2026-09-18T10:13:05.173Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"65389991-1854-4e00-a3fe-208529240aad","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.173Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"65389991-1854-4e00-a3fe-208529240aad","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.174Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"65389991-1854-4e00-a3fe-208529240aad","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.183Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"fb699be6-ab0c-4929-b57d-64f4ff7c819d","name":"ray-cluster-head"}
{"level":"error","ts":"2026-09-18T10:13:05.187Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d80bba21-fa44-4e56-a796-216d9270d936","error":"certificates.cert-manager.io \"ray-worker-cert-ray-cluster\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:13:05.187Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d80bba21-fa44-4e56-a796-216d9270d936","error":"certificates.cert-manager.io \"ray-worker-cert-ray-cluster\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.187Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d80bba21-fa44-4e56-a796-216d9270d936"}
{"level":"error","ts":"2026-09-18T10:13:05.187Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d80bba21-fa44-4e56-a796-216d9270d936","error":"certificates.cert-manager.io \"ray-worker-cert-ray-cluster\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.187Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"6cad0f47-c23e-4cbc-82bd-5b005b5a7b71","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.187Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"6cad0f47-c23e-4cbc-82bd-5b005b5a7b71"}
{"level":"info","ts":"2026-09-18T10:13:05.188Z","logger":"controllers.RayCluster","msg":"Created role for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4b3c500f-4f2f-4505-a545-c49f1e17054e","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.188Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"65389991-1854-4e00-a3fe-208529240aad","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.266Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"858132b6-e140-4587-b760-075503345484","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.266Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"858132b6-e140-4587-b760-075503345484"}
{"level":"info","ts":"2026-09-18T10:13:05.266Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"3210ac6c-db6b-43c6-a587-00c2771e0c7a","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.266Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"3210ac6c-db6b-43c6-a587-00c2771e0c7a"}
{"level":"info","ts":"2026-09-18T10:13:05.275Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"fb699be6-ab0c-4929-b57d-64f4ff7c819d","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:13:05.275Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"fb699be6-ab0c-4929-b57d-64f4ff7c819d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.275Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b819a728-d433-4ebf-a704-0e014a636d20","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.290Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b819a728-d433-4ebf-a704-0e014a636d20","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:13:05.293Z","logger":"controllers.RayCluster","msg":"Created role binding for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4b3c500f-4f2f-4505-a545-c49f1e17054e","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.293Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4b3c500f-4f2f-4505-a545-c49f1e17054e"}
{"level":"info","ts":"2026-09-18T10:13:05.293Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4b3c500f-4f2f-4505-a545-c49f1e17054e"}
{"level":"info","ts":"2026-09-18T10:13:05.293Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4b3c500f-4f2f-4505-a545-c49f1e17054e"}
{"level":"info","ts":"2026-09-18T10:13:05.304Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4b3c500f-4f2f-4505-a545-c49f1e17054e","name":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:05.304Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4b3c500f-4f2f-4505-a545-c49f1e17054e","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:13:05.304Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4b3c500f-4f2f-4505-a545-c49f1e17054e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.304Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4b3c500f-4f2f-4505-a545-c49f1e17054e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:13:05.304Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4b3c500f-4f2f-4505-a545-c49f1e17054e","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.304Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4b3c500f-4f2f-4505-a545-c49f1e17054e","serviceSelectors":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/identifier":"ray-cluster-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.304Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4b3c500f-4f2f-4505-a545-c49f1e17054e","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.304Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4b3c500f-4f2f-4505-a545-c49f1e17054e","error":"unable to find head service. cluster name ray-cluster, filter labels [test-ns-pmtv8 map[ray.io/cluster:ray-cluster ray.io/identifier:ray-cluster-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-09-18T10:13:05.304Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4b3c500f-4f2f-4505-a545-c49f1e17054e"}
{"level":"error","ts":"2026-09-18T10:13:05.304Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4b3c500f-4f2f-4505-a545-c49f1e17054e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.306Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b3623fe0-7b18-4207-ab79-b35612f3b13f","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pmtv8"}}
{"level":"info","ts":"2026-09-18T10:13:05.306Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b3623fe0-7b18-4207-ab79-b35612f3b13f","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.306Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b3623fe0-7b18-4207-ab79-b35612f3b13f","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.306Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4d389d26-490d-4df3-8504-b7b90933ca32"}
{"level":"info","ts":"2026-09-18T10:13:05.306Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4d389d26-490d-4df3-8504-b7b90933ca32"}
{"level":"info","ts":"2026-09-18T10:13:05.306Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4d389d26-490d-4df3-8504-b7b90933ca32"}
{"level":"info","ts":"2026-09-18T10:13:05.306Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4d389d26-490d-4df3-8504-b7b90933ca32","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:05.306Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4d389d26-490d-4df3-8504-b7b90933ca32","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:13:05.306Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4d389d26-490d-4df3-8504-b7b90933ca32","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.306Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4d389d26-490d-4df3-8504-b7b90933ca32","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:13:05.306Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4d389d26-490d-4df3-8504-b7b90933ca32","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.306Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4d389d26-490d-4df3-8504-b7b90933ca32","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.306Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4d389d26-490d-4df3-8504-b7b90933ca32","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.306Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4d389d26-490d-4df3-8504-b7b90933ca32","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"}]},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:13:05.312Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b819a728-d433-4ebf-a704-0e014a636d20","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:13:05.312Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b819a728-d433-4ebf-a704-0e014a636d20","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.328Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4d389d26-490d-4df3-8504-b7b90933ca32"}
{"level":"error","ts":"2026-09-18T10:13:05.328Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4d389d26-490d-4df3-8504-b7b90933ca32","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.328Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"55fb1e22-c2d4-403f-a7f1-ed03fe056977"}
{"level":"info","ts":"2026-09-18T10:13:05.328Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"55fb1e22-c2d4-403f-a7f1-ed03fe056977"}
{"level":"info","ts":"2026-09-18T10:13:05.328Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"55fb1e22-c2d4-403f-a7f1-ed03fe056977"}
{"level":"info","ts":"2026-09-18T10:13:05.328Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"55fb1e22-c2d4-403f-a7f1-ed03fe056977","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:05.328Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"55fb1e22-c2d4-403f-a7f1-ed03fe056977","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:13:05.328Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"55fb1e22-c2d4-403f-a7f1-ed03fe056977","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.328Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"55fb1e22-c2d4-403f-a7f1-ed03fe056977","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:13:05.328Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"55fb1e22-c2d4-403f-a7f1-ed03fe056977","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.328Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"55fb1e22-c2d4-403f-a7f1-ed03fe056977","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.328Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"55fb1e22-c2d4-403f-a7f1-ed03fe056977","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.328Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"55fb1e22-c2d4-403f-a7f1-ed03fe056977","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"}]},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:13:05.328Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"db13a393-065d-4149-b504-265917a6bc24","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.328Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"425ed9be-647e-4544-8bc0-e0ddb85b9685","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.328Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"425ed9be-647e-4544-8bc0-e0ddb85b9685"}
{"level":"info","ts":"2026-09-18T10:13:05.366Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"55fb1e22-c2d4-403f-a7f1-ed03fe056977","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pmtv8","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:13:05.366Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"55fb1e22-c2d4-403f-a7f1-ed03fe056977"}
{"level":"error","ts":"2026-09-18T10:13:05.366Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"55fb1e22-c2d4-403f-a7f1-ed03fe056977","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.366Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7d8b2633-c300-4486-9a76-55b7e3d6c95d"}
{"level":"info","ts":"2026-09-18T10:13:05.366Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7d8b2633-c300-4486-9a76-55b7e3d6c95d"}
{"level":"info","ts":"2026-09-18T10:13:05.366Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7d8b2633-c300-4486-9a76-55b7e3d6c95d"}
{"level":"info","ts":"2026-09-18T10:13:05.366Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7d8b2633-c300-4486-9a76-55b7e3d6c95d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:05.366Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7d8b2633-c300-4486-9a76-55b7e3d6c95d","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:13:05.366Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7d8b2633-c300-4486-9a76-55b7e3d6c95d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.366Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7d8b2633-c300-4486-9a76-55b7e3d6c95d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:13:05.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7d8b2633-c300-4486-9a76-55b7e3d6c95d","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7d8b2633-c300-4486-9a76-55b7e3d6c95d","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7d8b2633-c300-4486-9a76-55b7e3d6c95d","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.366Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7d8b2633-c300-4486-9a76-55b7e3d6c95d"}
{"level":"error","ts":"2026-09-18T10:13:05.366Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7d8b2633-c300-4486-9a76-55b7e3d6c95d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.373Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"db13a393-065d-4149-b504-265917a6bc24","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:13:05.386Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ac9830f2-fd5a-433c-8dba-2905d7ddaf04"}
{"level":"info","ts":"2026-09-18T10:13:05.386Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ac9830f2-fd5a-433c-8dba-2905d7ddaf04"}
{"level":"info","ts":"2026-09-18T10:13:05.386Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ac9830f2-fd5a-433c-8dba-2905d7ddaf04"}
{"level":"info","ts":"2026-09-18T10:13:05.386Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ac9830f2-fd5a-433c-8dba-2905d7ddaf04","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:05.387Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ac9830f2-fd5a-433c-8dba-2905d7ddaf04","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:13:05.387Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ac9830f2-fd5a-433c-8dba-2905d7ddaf04","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.387Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ac9830f2-fd5a-433c-8dba-2905d7ddaf04","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:13:05.387Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ac9830f2-fd5a-433c-8dba-2905d7ddaf04","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.387Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ac9830f2-fd5a-433c-8dba-2905d7ddaf04","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.387Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ac9830f2-fd5a-433c-8dba-2905d7ddaf04","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.387Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ac9830f2-fd5a-433c-8dba-2905d7ddaf04"}
{"level":"error","ts":"2026-09-18T10:13:05.387Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ac9830f2-fd5a-433c-8dba-2905d7ddaf04","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.389Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"db13a393-065d-4149-b504-265917a6bc24","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:13:05.389Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"db13a393-065d-4149-b504-265917a6bc24","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.395Z","logger":"authentication-controller","msg":"Added auth proxy SA to autoscaler RoleBinding","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b3623fe0-7b18-4207-ab79-b35612f3b13f","roleBinding":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"authServiceAccount":"ray-cluster-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:13:05.403Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b3623fe0-7b18-4207-ab79-b35612f3b13f","name":"kuberay-gateway-access","namespace":"test-ns-pmtv8","operation":"created"}
{"level":"info","ts":"2026-09-18T10:13:05.421Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b3623fe0-7b18-4207-ab79-b35612f3b13f","name":"test-ns-pmtv8-ray-cluster","operation":"created"}
{"level":"info","ts":"2026-09-18T10:13:05.430Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b3623fe0-7b18-4207-ab79-b35612f3b13f","name":"kube-rbac-proxy-config-ray-cluster","operation":"created"}
{"level":"error","ts":"2026-09-18T10:13:05.439Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b3623fe0-7b18-4207-ab79-b35612f3b13f","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.439Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b3623fe0-7b18-4207-ab79-b35612f3b13f"}
{"level":"error","ts":"2026-09-18T10:13:05.439Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b3623fe0-7b18-4207-ab79-b35612f3b13f","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.444Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"34046e83-af27-4b1c-9f61-4b226017829e","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pmtv8"}}
{"level":"info","ts":"2026-09-18T10:13:05.444Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"34046e83-af27-4b1c-9f61-4b226017829e","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.444Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"34046e83-af27-4b1c-9f61-4b226017829e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.460Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"74395a9e-3075-47c0-a1c5-e547d8f34c3d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.460Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d7be7a4d-0abd-4dba-a728-b490642db6da","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.460Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d7be7a4d-0abd-4dba-a728-b490642db6da"}
{"level":"info","ts":"2026-09-18T10:13:05.461Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"34046e83-af27-4b1c-9f61-4b226017829e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.467Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4f90b3b3-c594-46e3-9c49-613b29787032"}
{"level":"info","ts":"2026-09-18T10:13:05.467Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4f90b3b3-c594-46e3-9c49-613b29787032"}
{"level":"info","ts":"2026-09-18T10:13:05.467Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4f90b3b3-c594-46e3-9c49-613b29787032"}
{"level":"info","ts":"2026-09-18T10:13:05.467Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4f90b3b3-c594-46e3-9c49-613b29787032","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:05.467Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4f90b3b3-c594-46e3-9c49-613b29787032","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:13:05.467Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4f90b3b3-c594-46e3-9c49-613b29787032","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.467Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4f90b3b3-c594-46e3-9c49-613b29787032","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:13:05.468Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4f90b3b3-c594-46e3-9c49-613b29787032","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.468Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4f90b3b3-c594-46e3-9c49-613b29787032","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.468Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4f90b3b3-c594-46e3-9c49-613b29787032","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.468Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4f90b3b3-c594-46e3-9c49-613b29787032"}
{"level":"error","ts":"2026-09-18T10:13:05.468Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4f90b3b3-c594-46e3-9c49-613b29787032","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.477Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"74395a9e-3075-47c0-a1c5-e547d8f34c3d","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:13:05.491Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"74395a9e-3075-47c0-a1c5-e547d8f34c3d","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:13:05.491Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"74395a9e-3075-47c0-a1c5-e547d8f34c3d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:05.628Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"89ca60af-7255-4124-8888-672dbe943e97"}
{"level":"info","ts":"2026-09-18T10:13:05.628Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"89ca60af-7255-4124-8888-672dbe943e97"}
{"level":"info","ts":"2026-09-18T10:13:05.628Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"89ca60af-7255-4124-8888-672dbe943e97"}
{"level":"info","ts":"2026-09-18T10:13:05.628Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"89ca60af-7255-4124-8888-672dbe943e97","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:05.628Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"89ca60af-7255-4124-8888-672dbe943e97","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:13:05.628Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"89ca60af-7255-4124-8888-672dbe943e97","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.628Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"89ca60af-7255-4124-8888-672dbe943e97","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:13:05.628Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"89ca60af-7255-4124-8888-672dbe943e97","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.628Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"89ca60af-7255-4124-8888-672dbe943e97","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.628Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"89ca60af-7255-4124-8888-672dbe943e97","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.628Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"89ca60af-7255-4124-8888-672dbe943e97"}
{"level":"error","ts":"2026-09-18T10:13:05.628Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"89ca60af-7255-4124-8888-672dbe943e97","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.949Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"691abb63-fc10-46c6-9d22-cee816fb1d80"}
{"level":"info","ts":"2026-09-18T10:13:05.949Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"691abb63-fc10-46c6-9d22-cee816fb1d80"}
{"level":"info","ts":"2026-09-18T10:13:05.949Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"691abb63-fc10-46c6-9d22-cee816fb1d80"}
{"level":"info","ts":"2026-09-18T10:13:05.949Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"691abb63-fc10-46c6-9d22-cee816fb1d80","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:05.949Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"691abb63-fc10-46c6-9d22-cee816fb1d80","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:13:05.949Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"691abb63-fc10-46c6-9d22-cee816fb1d80","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:05.949Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"691abb63-fc10-46c6-9d22-cee816fb1d80","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:13:05.949Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"691abb63-fc10-46c6-9d22-cee816fb1d80","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.949Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"691abb63-fc10-46c6-9d22-cee816fb1d80","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.949Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"691abb63-fc10-46c6-9d22-cee816fb1d80","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:05.949Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"691abb63-fc10-46c6-9d22-cee816fb1d80"}
{"level":"error","ts":"2026-09-18T10:13:05.949Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"691abb63-fc10-46c6-9d22-cee816fb1d80","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:06.590Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2dd8df5b-03ba-4b2f-b471-11a7167a5ba9"}
{"level":"info","ts":"2026-09-18T10:13:06.590Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2dd8df5b-03ba-4b2f-b471-11a7167a5ba9"}
{"level":"info","ts":"2026-09-18T10:13:06.590Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2dd8df5b-03ba-4b2f-b471-11a7167a5ba9"}
{"level":"info","ts":"2026-09-18T10:13:06.590Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2dd8df5b-03ba-4b2f-b471-11a7167a5ba9","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:06.590Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2dd8df5b-03ba-4b2f-b471-11a7167a5ba9","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:13:06.590Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2dd8df5b-03ba-4b2f-b471-11a7167a5ba9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:06.590Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2dd8df5b-03ba-4b2f-b471-11a7167a5ba9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:13:06.590Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2dd8df5b-03ba-4b2f-b471-11a7167a5ba9","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:06.590Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2dd8df5b-03ba-4b2f-b471-11a7167a5ba9","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:06.590Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2dd8df5b-03ba-4b2f-b471-11a7167a5ba9","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:06.590Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2dd8df5b-03ba-4b2f-b471-11a7167a5ba9"}
{"level":"error","ts":"2026-09-18T10:13:06.590Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2dd8df5b-03ba-4b2f-b471-11a7167a5ba9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:07.870Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b"}
{"level":"info","ts":"2026-09-18T10:13:07.870Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b"}
{"level":"info","ts":"2026-09-18T10:13:07.870Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b"}
{"level":"info","ts":"2026-09-18T10:13:07.870Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:07.871Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b"}
{"level":"info","ts":"2026-09-18T10:13:07.871Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:13:07.871Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b"}
{"level":"info","ts":"2026-09-18T10:13:07.871Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:07.871Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:13:07.871Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:13:07.871Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:13:07.871Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:13:07.871Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:13:07.871Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:13:07.871Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","enabled":true}
{"level":"info","ts":"2026-09-18T10:13:07.895Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","name":"ray-cluster-head-w667z"}
{"level":"info","ts":"2026-09-18T10:13:07.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:13:07.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:07.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:07.895Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:07.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:07.895Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:07.895Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:07.895Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"42ce4a5c-9305-45a2-840d-efb33b35c27b","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:13:07.915Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303"}
{"level":"info","ts":"2026-09-18T10:13:07.915Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303"}
{"level":"info","ts":"2026-09-18T10:13:07.915Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303"}
{"level":"info","ts":"2026-09-18T10:13:07.915Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:07.915Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"875c49f7-590d-4021-b43d-7238221daaf4","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:07.915Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:07.915Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:07.915Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:13:07.915Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:07.915Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:07.915Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:07.915Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:07.915Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:07.915Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"875c49f7-590d-4021-b43d-7238221daaf4","rayCluster":"ray-cluster","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:13:07.915Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:07.916Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d86dd7f2-f4ca-45f3-bd91-ff7b0c14e12e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:07.916Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:13:07.924Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pmtv8","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:13:07.924Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303"}
{"level":"error","ts":"2026-09-18T10:13:07.924Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"15bc58f4-f9eb-4b70-aa19-6d4aa667e303","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:07.929Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fd85099c-4ee6-4edc-bf19-e81ef0243739"}
{"level":"info","ts":"2026-09-18T10:13:07.929Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fd85099c-4ee6-4edc-bf19-e81ef0243739"}
{"level":"info","ts":"2026-09-18T10:13:07.929Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fd85099c-4ee6-4edc-bf19-e81ef0243739"}
{"level":"info","ts":"2026-09-18T10:13:07.929Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fd85099c-4ee6-4edc-bf19-e81ef0243739","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:07.929Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d86dd7f2-f4ca-45f3-bd91-ff7b0c14e12e","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:13:07.929Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fd85099c-4ee6-4edc-bf19-e81ef0243739","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:07.929Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fd85099c-4ee6-4edc-bf19-e81ef0243739","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:07.929Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fd85099c-4ee6-4edc-bf19-e81ef0243739","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:13:07.929Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fd85099c-4ee6-4edc-bf19-e81ef0243739","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:07.929Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fd85099c-4ee6-4edc-bf19-e81ef0243739","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:07.929Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fd85099c-4ee6-4edc-bf19-e81ef0243739","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:07.929Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fd85099c-4ee6-4edc-bf19-e81ef0243739","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:07.929Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fd85099c-4ee6-4edc-bf19-e81ef0243739","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:07.929Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fd85099c-4ee6-4edc-bf19-e81ef0243739","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fd85099c-4ee6-4edc-bf19-e81ef0243739","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fd85099c-4ee6-4edc-bf19-e81ef0243739","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7df63e0a-5d7c-41c5-9718-865859462d88"}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7df63e0a-5d7c-41c5-9718-865859462d88"}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7df63e0a-5d7c-41c5-9718-865859462d88"}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7df63e0a-5d7c-41c5-9718-865859462d88","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7df63e0a-5d7c-41c5-9718-865859462d88","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7df63e0a-5d7c-41c5-9718-865859462d88","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7df63e0a-5d7c-41c5-9718-865859462d88","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7df63e0a-5d7c-41c5-9718-865859462d88","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7df63e0a-5d7c-41c5-9718-865859462d88","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7df63e0a-5d7c-41c5-9718-865859462d88","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7df63e0a-5d7c-41c5-9718-865859462d88","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7df63e0a-5d7c-41c5-9718-865859462d88","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7df63e0a-5d7c-41c5-9718-865859462d88","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7df63e0a-5d7c-41c5-9718-865859462d88","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:07.930Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7df63e0a-5d7c-41c5-9718-865859462d88","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:07.942Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d86dd7f2-f4ca-45f3-bd91-ff7b0c14e12e","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:13:07.942Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d86dd7f2-f4ca-45f3-bd91-ff7b0c14e12e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:07.954Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fca2716a-1d45-42e3-a75a-00b0eafb72b3"}
{"level":"info","ts":"2026-09-18T10:13:07.954Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fca2716a-1d45-42e3-a75a-00b0eafb72b3"}
{"level":"info","ts":"2026-09-18T10:13:07.954Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fca2716a-1d45-42e3-a75a-00b0eafb72b3"}
{"level":"info","ts":"2026-09-18T10:13:07.954Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fca2716a-1d45-42e3-a75a-00b0eafb72b3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:07.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fca2716a-1d45-42e3-a75a-00b0eafb72b3","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:07.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fca2716a-1d45-42e3-a75a-00b0eafb72b3","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:07.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fca2716a-1d45-42e3-a75a-00b0eafb72b3","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:13:07.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fca2716a-1d45-42e3-a75a-00b0eafb72b3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:07.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fca2716a-1d45-42e3-a75a-00b0eafb72b3","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:07.954Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fca2716a-1d45-42e3-a75a-00b0eafb72b3","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:07.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fca2716a-1d45-42e3-a75a-00b0eafb72b3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:07.954Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fca2716a-1d45-42e3-a75a-00b0eafb72b3","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:07.954Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fca2716a-1d45-42e3-a75a-00b0eafb72b3","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:07.954Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fca2716a-1d45-42e3-a75a-00b0eafb72b3","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]; autoscaler: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:13:07.967Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"a3c4d48b-9e33-40f9-adf5-769297cff69a","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:07.967Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d98a6d40-0b46-404f-b373-3945dc7d5f5a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:07.967Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"a3c4d48b-9e33-40f9-adf5-769297cff69a","rayCluster":"ray-cluster","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:13:07.978Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d98a6d40-0b46-404f-b373-3945dc7d5f5a","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:13:07.993Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d98a6d40-0b46-404f-b373-3945dc7d5f5a","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:13:07.993Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d98a6d40-0b46-404f-b373-3945dc7d5f5a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:08.185Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"04757df4-2aba-4fc4-9fdf-eb1710627ea0"}
{"level":"info","ts":"2026-09-18T10:13:08.185Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"04757df4-2aba-4fc4-9fdf-eb1710627ea0"}
{"level":"info","ts":"2026-09-18T10:13:08.185Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"04757df4-2aba-4fc4-9fdf-eb1710627ea0"}
{"level":"info","ts":"2026-09-18T10:13:08.185Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"04757df4-2aba-4fc4-9fdf-eb1710627ea0","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:08.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"04757df4-2aba-4fc4-9fdf-eb1710627ea0","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:08.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"04757df4-2aba-4fc4-9fdf-eb1710627ea0","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:08.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"04757df4-2aba-4fc4-9fdf-eb1710627ea0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:08.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"04757df4-2aba-4fc4-9fdf-eb1710627ea0","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:08.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"04757df4-2aba-4fc4-9fdf-eb1710627ea0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:08.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"04757df4-2aba-4fc4-9fdf-eb1710627ea0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:08.186Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"04757df4-2aba-4fc4-9fdf-eb1710627ea0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:13:08.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"04757df4-2aba-4fc4-9fdf-eb1710627ea0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:08.186Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"04757df4-2aba-4fc4-9fdf-eb1710627ea0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:08.186Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"04757df4-2aba-4fc4-9fdf-eb1710627ea0","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:08.192Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"41eeef3d-9bac-491c-bcd5-ee773c18dcdb"}
{"level":"info","ts":"2026-09-18T10:13:08.192Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"41eeef3d-9bac-491c-bcd5-ee773c18dcdb"}
{"level":"info","ts":"2026-09-18T10:13:08.193Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"41eeef3d-9bac-491c-bcd5-ee773c18dcdb"}
{"level":"info","ts":"2026-09-18T10:13:08.193Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"41eeef3d-9bac-491c-bcd5-ee773c18dcdb","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:08.193Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"41eeef3d-9bac-491c-bcd5-ee773c18dcdb","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:08.193Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"41eeef3d-9bac-491c-bcd5-ee773c18dcdb","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:08.193Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"41eeef3d-9bac-491c-bcd5-ee773c18dcdb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:08.193Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"41eeef3d-9bac-491c-bcd5-ee773c18dcdb","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:08.193Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"41eeef3d-9bac-491c-bcd5-ee773c18dcdb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:08.193Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"41eeef3d-9bac-491c-bcd5-ee773c18dcdb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:08.193Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"41eeef3d-9bac-491c-bcd5-ee773c18dcdb","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:13:08.193Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"41eeef3d-9bac-491c-bcd5-ee773c18dcdb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:08.193Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"41eeef3d-9bac-491c-bcd5-ee773c18dcdb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:08.193Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"41eeef3d-9bac-491c-bcd5-ee773c18dcdb","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:08.217Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"324868df-4e7d-4c38-b713-09fc5744190a"}
{"level":"info","ts":"2026-09-18T10:13:08.217Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"324868df-4e7d-4c38-b713-09fc5744190a"}
{"level":"info","ts":"2026-09-18T10:13:08.217Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"324868df-4e7d-4c38-b713-09fc5744190a"}
{"level":"info","ts":"2026-09-18T10:13:08.217Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"324868df-4e7d-4c38-b713-09fc5744190a","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:08.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"324868df-4e7d-4c38-b713-09fc5744190a","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:08.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"324868df-4e7d-4c38-b713-09fc5744190a","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:08.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"324868df-4e7d-4c38-b713-09fc5744190a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:08.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"324868df-4e7d-4c38-b713-09fc5744190a","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:08.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"324868df-4e7d-4c38-b713-09fc5744190a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:08.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"324868df-4e7d-4c38-b713-09fc5744190a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:08.218Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"324868df-4e7d-4c38-b713-09fc5744190a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:13:08.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"324868df-4e7d-4c38-b713-09fc5744190a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:08.218Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"324868df-4e7d-4c38-b713-09fc5744190a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:08.218Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"324868df-4e7d-4c38-b713-09fc5744190a","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:08.230Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8ed6a47f-d87a-4010-88ea-28197f733a35"}
{"level":"info","ts":"2026-09-18T10:13:08.230Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8ed6a47f-d87a-4010-88ea-28197f733a35"}
{"level":"info","ts":"2026-09-18T10:13:08.230Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8ed6a47f-d87a-4010-88ea-28197f733a35"}
{"level":"info","ts":"2026-09-18T10:13:08.230Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8ed6a47f-d87a-4010-88ea-28197f733a35","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:08.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8ed6a47f-d87a-4010-88ea-28197f733a35","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:08.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8ed6a47f-d87a-4010-88ea-28197f733a35","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:08.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8ed6a47f-d87a-4010-88ea-28197f733a35","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:08.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8ed6a47f-d87a-4010-88ea-28197f733a35","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5qfl8-small-group-worker-tbqtj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:08.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8ed6a47f-d87a-4010-88ea-28197f733a35","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:08.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8ed6a47f-d87a-4010-88ea-28197f733a35","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:08.230Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8ed6a47f-d87a-4010-88ea-28197f733a35","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:13:08.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8ed6a47f-d87a-4010-88ea-28197f733a35","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:08.230Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8ed6a47f-d87a-4010-88ea-28197f733a35","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:08.230Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8ed6a47f-d87a-4010-88ea-28197f733a35","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:08.505Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"395eb87b-e778-4ef9-8dca-7f66ebaf03f3"}
{"level":"info","ts":"2026-09-18T10:13:08.505Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"395eb87b-e778-4ef9-8dca-7f66ebaf03f3"}
{"level":"info","ts":"2026-09-18T10:13:08.505Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"395eb87b-e778-4ef9-8dca-7f66ebaf03f3"}
{"level":"info","ts":"2026-09-18T10:13:08.505Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"395eb87b-e778-4ef9-8dca-7f66ebaf03f3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:08.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"395eb87b-e778-4ef9-8dca-7f66ebaf03f3","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:08.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"395eb87b-e778-4ef9-8dca-7f66ebaf03f3","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:08.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"395eb87b-e778-4ef9-8dca-7f66ebaf03f3","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:13:08.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"395eb87b-e778-4ef9-8dca-7f66ebaf03f3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:08.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"395eb87b-e778-4ef9-8dca-7f66ebaf03f3","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:08.505Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"395eb87b-e778-4ef9-8dca-7f66ebaf03f3","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:08.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"395eb87b-e778-4ef9-8dca-7f66ebaf03f3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:08.505Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"395eb87b-e778-4ef9-8dca-7f66ebaf03f3","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:08.505Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"395eb87b-e778-4ef9-8dca-7f66ebaf03f3","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:08.505Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"395eb87b-e778-4ef9-8dca-7f66ebaf03f3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:08.505Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"395eb87b-e778-4ef9-8dca-7f66ebaf03f3","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:08.762Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310c86b7-bb25-4508-8fcb-7fd83680cab8"}
{"level":"info","ts":"2026-09-18T10:13:08.762Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310c86b7-bb25-4508-8fcb-7fd83680cab8"}
{"level":"info","ts":"2026-09-18T10:13:08.762Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310c86b7-bb25-4508-8fcb-7fd83680cab8"}
{"level":"info","ts":"2026-09-18T10:13:08.762Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310c86b7-bb25-4508-8fcb-7fd83680cab8","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:08.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310c86b7-bb25-4508-8fcb-7fd83680cab8","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:08.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310c86b7-bb25-4508-8fcb-7fd83680cab8","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:08.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310c86b7-bb25-4508-8fcb-7fd83680cab8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:08.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310c86b7-bb25-4508-8fcb-7fd83680cab8","worker Pod":"long-running-5qfl8-small-group-worker-tbqtj","shouldDelete":true,"reason":"The worker Pod long-running-5qfl8-small-group-worker-tbqtj status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-09-18T10:13:08.768Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310c86b7-bb25-4508-8fcb-7fd83680cab8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:08.768Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310c86b7-bb25-4508-8fcb-7fd83680cab8","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-09-18T10:13:08.768Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310c86b7-bb25-4508-8fcb-7fd83680cab8","name":"long-running-5qfl8","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:05:17Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.42","serviceIP":"10.133.0.42","podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:37Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.42","serviceIP":"10.133.0.42","podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:37Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:13:08.788Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"147b1f20-61ee-4f97-95d7-6cd1bfcb453b","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:13:08.788Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"147b1f20-61ee-4f97-95d7-6cd1bfcb453b","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-09-18 10:03:20 +0000 UTC","Now":"2026-09-18T10:13:08.788Z","ShutdownTime":"2026-09-18T10:13:20.000Z"}
{"level":"info","ts":"2026-09-18T10:13:08.788Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"147b1f20-61ee-4f97-95d7-6cd1bfcb453b","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-09-18T10:13:20.000Z","requeueAfter":13.211784359}
{"level":"info","ts":"2026-09-18T10:13:08.788Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310c86b7-bb25-4508-8fcb-7fd83680cab8"}
{"level":"error","ts":"2026-09-18T10:13:08.788Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"310c86b7-bb25-4508-8fcb-7fd83680cab8","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:08.788Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"d7eefd2c-949d-435b-9837-c64215f1de64","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:08.788Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"12456a70-93ea-4027-9169-1b4ae9ca88b2","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:08.788Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"d7eefd2c-949d-435b-9837-c64215f1de64","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:13:08.788Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"d7eefd2c-949d-435b-9837-c64215f1de64","pod":"long-running-5qfl8-small-group-worker-tbqtj","ip":"10.132.0.47"}
{"level":"info","ts":"2026-09-18T10:13:08.788Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"d7eefd2c-949d-435b-9837-c64215f1de64","rayCluster":"long-running-5qfl8","podIPs":["10.133.0.42","10.132.0.47"]}
{"level":"info","ts":"2026-09-18T10:13:08.793Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ddc67411-306a-462e-bb43-3f4b4b9056fd"}
{"level":"info","ts":"2026-09-18T10:13:08.793Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ddc67411-306a-462e-bb43-3f4b4b9056fd"}
{"level":"info","ts":"2026-09-18T10:13:08.793Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ddc67411-306a-462e-bb43-3f4b4b9056fd"}
{"level":"info","ts":"2026-09-18T10:13:08.793Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ddc67411-306a-462e-bb43-3f4b4b9056fd","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:08.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ddc67411-306a-462e-bb43-3f4b4b9056fd","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:08.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ddc67411-306a-462e-bb43-3f4b4b9056fd","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:08.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ddc67411-306a-462e-bb43-3f4b4b9056fd","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:13:08.793Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ddc67411-306a-462e-bb43-3f4b4b9056fd","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-09-18T10:13:08.793Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ddc67411-306a-462e-bb43-3f4b4b9056fd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:08.793Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ddc67411-306a-462e-bb43-3f4b4b9056fd","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:08.796Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"12456a70-93ea-4027-9169-1b4ae9ca88b2","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:09.162Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a1239a70-4b4a-4319-8db4-9f4154a4aed7"}
{"level":"info","ts":"2026-09-18T10:13:09.162Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a1239a70-4b4a-4319-8db4-9f4154a4aed7"}
{"level":"info","ts":"2026-09-18T10:13:09.162Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a1239a70-4b4a-4319-8db4-9f4154a4aed7"}
{"level":"info","ts":"2026-09-18T10:13:09.162Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a1239a70-4b4a-4319-8db4-9f4154a4aed7","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:09.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a1239a70-4b4a-4319-8db4-9f4154a4aed7","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:09.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a1239a70-4b4a-4319-8db4-9f4154a4aed7","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:09.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a1239a70-4b4a-4319-8db4-9f4154a4aed7","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:13:09.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a1239a70-4b4a-4319-8db4-9f4154a4aed7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a1239a70-4b4a-4319-8db4-9f4154a4aed7","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:09.163Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a1239a70-4b4a-4319-8db4-9f4154a4aed7","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:09.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a1239a70-4b4a-4319-8db4-9f4154a4aed7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.163Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a1239a70-4b4a-4319-8db4-9f4154a4aed7","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:09.163Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a1239a70-4b4a-4319-8db4-9f4154a4aed7","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:09.163Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a1239a70-4b4a-4319-8db4-9f4154a4aed7","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]; autoscaler: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]; autoscaler: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:13:09.179Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"8785a9f5-c4ac-4ed0-b83e-1dd6f8dfce7a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:09.179Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"dc6d5268-e1b0-4a67-8c58-1821e50028a6","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:09.179Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"dc6d5268-e1b0-4a67-8c58-1821e50028a6","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:13:09.183Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dcd3f449-f466-44ba-b65c-5cd751409e10"}
{"level":"info","ts":"2026-09-18T10:13:09.183Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dcd3f449-f466-44ba-b65c-5cd751409e10"}
{"level":"info","ts":"2026-09-18T10:13:09.183Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dcd3f449-f466-44ba-b65c-5cd751409e10"}
{"level":"info","ts":"2026-09-18T10:13:09.183Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dcd3f449-f466-44ba-b65c-5cd751409e10","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:09.183Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dcd3f449-f466-44ba-b65c-5cd751409e10","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:09.183Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dcd3f449-f466-44ba-b65c-5cd751409e10","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:09.183Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dcd3f449-f466-44ba-b65c-5cd751409e10","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:13:09.184Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dcd3f449-f466-44ba-b65c-5cd751409e10","podName":"long-running-5qfl8-small-group-worker-tbqtj","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-09-18T10:13:09.184Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dcd3f449-f466-44ba-b65c-5cd751409e10","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:09.184Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dcd3f449-f466-44ba-b65c-5cd751409e10","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64"}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64"}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64"}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64"}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","nodeType":"worker","rayStartParams":{"address":"long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","rayStartCmd":"ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:09.185Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","enabled":true}
{"level":"error","ts":"2026-09-18T10:13:09.190Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.190Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:13:09.190Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","name":"long-running-5qfl8","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:08Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.42","serviceIP":"10.133.0.42","podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:37Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:09Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.42","serviceIP":"10.133.0.42","podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:37Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T10:13:09Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:13:09.195Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"8785a9f5-c4ac-4ed0-b83e-1dd6f8dfce7a","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:13:09.195Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"dc6d5268-e1b0-4a67-8c58-1821e50028a6","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-pmtv8.svc","ray-cluster-head-svc.test-ns-pmtv8.svc.cluster.local"],"ipAddresses":["10.132.0.59","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:13:09.207Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64"}
{"level":"error","ts":"2026-09-18T10:13:09.207Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"423b21b4-01bf-4983-b182-7b6a0d11cc64","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.207Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"3c9519e3-a652-497c-a80b-01673cbe17fd","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:13:09.207Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"3c9519e3-a652-497c-a80b-01673cbe17fd","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-09-18 10:03:20 +0000 UTC","Now":"2026-09-18T10:13:09.207Z","ShutdownTime":"2026-09-18T10:13:20.000Z"}
{"level":"info","ts":"2026-09-18T10:13:09.207Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"3c9519e3-a652-497c-a80b-01673cbe17fd","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-09-18T10:13:20.000Z","requeueAfter":12.792278152}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"8785a9f5-c4ac-4ed0-b83e-1dd6f8dfce7a","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"8785a9f5-c4ac-4ed0-b83e-1dd6f8dfce7a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"d533ff74-a4ff-4a64-a840-bdedce0eb28c","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb"}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb"}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb"}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb"}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb","nodeType":"worker","rayStartParams":{"address":"long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb","rayStartCmd":"ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:09.212Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb","enabled":true}
{"level":"info","ts":"2026-09-18T10:13:09.213Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"dc6d5268-e1b0-4a67-8c58-1821e50028a6","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-pmtv8.svc","ray-cluster-worker-svc.test-ns-pmtv8.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-pmtv8.svc","*.ray-cluster-worker-svc.test-ns-pmtv8.svc.cluster.local","*-worker-*.test-ns-pmtv8.svc","*-worker-*.test-ns-pmtv8.svc.cluster.local"],"ipAddresses":["10.132.0.59","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:13:09.213Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"dc6d5268-e1b0-4a67-8c58-1821e50028a6","rayCluster":"ray-cluster","podIPs":["10.132.0.59"]}
{"level":"info","ts":"2026-09-18T10:13:09.213Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"88369d6f-cb0f-4069-bf12-8a3b797cc505","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:09.213Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"88369d6f-cb0f-4069-bf12-8a3b797cc505","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:13:09.266Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"88369d6f-cb0f-4069-bf12-8a3b797cc505","certificate":"ray-head-cert-long-running-5qfl8","dnsNames":["long-running-5qfl8-head-svc","localhost","long-running-5qfl8-head-svc.test-ns-ltx56.svc","long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.42","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:13:09.270Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"d533ff74-a4ff-4a64-a840-bdedce0eb28c","cluster":"long-running-5qfl8"}
{"level":"error","ts":"2026-09-18T10:13:09.270Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.270Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:13:09.270Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb"}
{"level":"error","ts":"2026-09-18T10:13:09.270Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"09ac416d-78c1-4820-8093-d0a324eae4bb","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795"}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795"}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795"}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795"}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795","nodeType":"worker","rayStartParams":{"address":"long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795","rayStartCmd":"ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795","enabled":true}
{"level":"info","ts":"2026-09-18T10:13:09.281Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"88369d6f-cb0f-4069-bf12-8a3b797cc505","certificate":"ray-worker-cert-long-running-5qfl8","dnsNames":["long-running-5qfl8-worker-svc","localhost","long-running-5qfl8-small-group","long-running-5qfl8-worker-svc.test-ns-ltx56.svc","long-running-5qfl8-worker-svc.test-ns-ltx56.svc.cluster.local","*.long-running-5qfl8-worker-svc.test-ns-ltx56.svc","*.long-running-5qfl8-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["10.133.0.42","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:13:09.282Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"88369d6f-cb0f-4069-bf12-8a3b797cc505"}
{"level":"error","ts":"2026-09-18T10:13:09.285Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.285Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:13:09.286Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795"}
{"level":"error","ts":"2026-09-18T10:13:09.286Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"113fbcb0-8088-4107-8707-d799652a9795","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf"}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf"}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf"}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf"}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf","nodeType":"worker","rayStartParams":{"address":"long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf","rayStartCmd":"ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:09.307Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf","enabled":true}
{"level":"error","ts":"2026-09-18T10:13:09.366Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.366Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:13:09.366Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf"}
{"level":"error","ts":"2026-09-18T10:13:09.366Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"115010c5-f89e-4295-a947-79e368dcd5cf","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.406Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385"}
{"level":"info","ts":"2026-09-18T10:13:09.406Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385"}
{"level":"info","ts":"2026-09-18T10:13:09.407Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385"}
{"level":"info","ts":"2026-09-18T10:13:09.407Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:09.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:09.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:09.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:09.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:13:09.407Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:09.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:13:09.407Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385"}
{"level":"info","ts":"2026-09-18T10:13:09.407Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385","nodeType":"worker","rayStartParams":{"address":"long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:13:09.407Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385","rayStartCmd":"ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:09.407Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:09.407Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385","enabled":true}
{"level":"error","ts":"2026-09-18T10:13:09.411Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.411Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:13:09.411Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385"}
{"level":"error","ts":"2026-09-18T10:13:09.411Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"2aeb054d-58a6-4532-a133-7a8f32f6c385","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.492Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1"}
{"level":"info","ts":"2026-09-18T10:13:09.492Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1"}
{"level":"info","ts":"2026-09-18T10:13:09.492Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1"}
{"level":"info","ts":"2026-09-18T10:13:09.492Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:09.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:09.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:09.493Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:09.493Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.493Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:13:09.493Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:09.493Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.493Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:13:09.493Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1"}
{"level":"info","ts":"2026-09-18T10:13:09.493Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1","nodeType":"worker","rayStartParams":{"address":"long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:13:09.493Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1","rayStartCmd":"ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:09.493Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:09.493Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1","enabled":true}
{"level":"error","ts":"2026-09-18T10:13:09.497Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.498Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:13:09.498Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1"}
{"level":"error","ts":"2026-09-18T10:13:09.498Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"8871f0a2-be33-4607-97be-7e60766c32c1","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8"}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8"}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8"}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5qfl8-head-kflfc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8"}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8","nodeType":"worker","rayStartParams":{"address":"long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8","rayStartCmd":"ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:09.659Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8","enabled":true}
{"level":"error","ts":"2026-09-18T10:13:09.663Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.663Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:13:09.664Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8"}
{"level":"error","ts":"2026-09-18T10:13:09.664Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"e94b8cf3-80b1-4f61-9e2d-af25068134e8","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.859Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"b8298f3b-91ba-409b-abc9-00de8472ee94"}
{"level":"info","ts":"2026-09-18T10:13:09.859Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"b8298f3b-91ba-409b-abc9-00de8472ee94"}
{"level":"info","ts":"2026-09-18T10:13:09.859Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"b8298f3b-91ba-409b-abc9-00de8472ee94"}
{"level":"info","ts":"2026-09-18T10:13:09.859Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"b8298f3b-91ba-409b-abc9-00de8472ee94","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:09.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"b8298f3b-91ba-409b-abc9-00de8472ee94","Found 1 head Pod":"long-running-5qfl8-head-kflfc","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-09-18 10:03:25 +0000 UTC,FinishedAt:2026-09-18 10:13:09 +0000 UTC,ContainerID:cri-o://ca644db758fbe8b0a93e0dbe2597b6a2144b7d6e5d8a57a35abb9ad8e0d724b5,}"}
{"level":"info","ts":"2026-09-18T10:13:09.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"b8298f3b-91ba-409b-abc9-00de8472ee94","head Pod":"long-running-5qfl8-head-kflfc","shouldDelete":true,"reason":"The head Pod long-running-5qfl8-head-kflfc status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-09-18T10:13:09.875Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"b8298f3b-91ba-409b-abc9-00de8472ee94","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod long-running-5qfl8-head-kflfc status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.875Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"b8298f3b-91ba-409b-abc9-00de8472ee94","error":"The head Pod long-running-5qfl8-head-kflfc status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-09-18T10:13:09.875Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"b8298f3b-91ba-409b-abc9-00de8472ee94","name":"long-running-5qfl8","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:09Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.42","serviceIP":"10.133.0.42","podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:03:37Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T10:13:09Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:09Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.42","serviceIP":"10.133.0.42","podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:13:09Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T10:13:09Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:13:09.893Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"b8298f3b-91ba-409b-abc9-00de8472ee94"}
{"level":"error","ts":"2026-09-18T10:13:09.893Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"b8298f3b-91ba-409b-abc9-00de8472ee94","error":"The head Pod long-running-5qfl8-head-kflfc status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.894Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"b798793d-63a1-4fd5-b781-be2fb67b69bb","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:09.894Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"75aa12ae-a784-41cd-8e2c-6baa39f97e1a","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:13:09.894Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"75aa12ae-a784-41cd-8e2c-6baa39f97e1a","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-09-18 10:03:20 +0000 UTC","Now":"2026-09-18T10:13:09.894Z","ShutdownTime":"2026-09-18T10:13:20.000Z"}
{"level":"info","ts":"2026-09-18T10:13:09.894Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"75aa12ae-a784-41cd-8e2c-6baa39f97e1a","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-09-18T10:13:20.000Z","requeueAfter":12.105379666}
{"level":"info","ts":"2026-09-18T10:13:09.894Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"534e9809-9769-460a-b9f9-4bff218da422","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:09.894Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"534e9809-9769-460a-b9f9-4bff218da422","pod":"long-running-5qfl8-head-kflfc","ip":"10.133.0.42"}
{"level":"info","ts":"2026-09-18T10:13:09.894Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"534e9809-9769-460a-b9f9-4bff218da422"}
{"level":"info","ts":"2026-09-18T10:13:09.900Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"b798793d-63a1-4fd5-b781-be2fb67b69bb","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:09.967Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"78ef9028-03e2-4936-a57a-68bde4c85c53"}
{"level":"info","ts":"2026-09-18T10:13:09.967Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"78ef9028-03e2-4936-a57a-68bde4c85c53"}
{"level":"info","ts":"2026-09-18T10:13:09.967Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"78ef9028-03e2-4936-a57a-68bde4c85c53"}
{"level":"info","ts":"2026-09-18T10:13:09.967Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"78ef9028-03e2-4936-a57a-68bde4c85c53","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:09.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"78ef9028-03e2-4936-a57a-68bde4c85c53","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:09.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"78ef9028-03e2-4936-a57a-68bde4c85c53","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:09.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"78ef9028-03e2-4936-a57a-68bde4c85c53","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:13:09.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"78ef9028-03e2-4936-a57a-68bde4c85c53","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"78ef9028-03e2-4936-a57a-68bde4c85c53","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:09.967Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"78ef9028-03e2-4936-a57a-68bde4c85c53","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:09.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"78ef9028-03e2-4936-a57a-68bde4c85c53","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.967Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"78ef9028-03e2-4936-a57a-68bde4c85c53","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:09.967Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"78ef9028-03e2-4936-a57a-68bde4c85c53","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:09.967Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"78ef9028-03e2-4936-a57a-68bde4c85c53","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:09.967Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"78ef9028-03e2-4936-a57a-68bde4c85c53","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:09.985Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c"}
{"level":"info","ts":"2026-09-18T10:13:09.985Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c"}
{"level":"info","ts":"2026-09-18T10:13:09.985Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c"}
{"level":"info","ts":"2026-09-18T10:13:09.985Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:09.985Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-09-18T10:13:09.985Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:09.985Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.985Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:13:09.985Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:09.985Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:09.985Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:13:09.985Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c"}
{"level":"info","ts":"2026-09-18T10:13:09.985Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c","nodeType":"worker","rayStartParams":{"address":"long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:13:09.985Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c","rayStartCmd":"ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:09.985Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:09.985Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c","enabled":true}
{"level":"error","ts":"2026-09-18T10:13:09.990Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:09.990Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:13:09.991Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c"}
{"level":"error","ts":"2026-09-18T10:13:09.991Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"5ee72a0d-c18c-491c-8d0b-de513e0db30c","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:10.224Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356"}
{"level":"info","ts":"2026-09-18T10:13:10.224Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356"}
{"level":"info","ts":"2026-09-18T10:13:10.224Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356"}
{"level":"info","ts":"2026-09-18T10:13:10.224Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:10.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-09-18T10:13:10.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:10.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:10.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:13:10.224Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:10.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:10.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:13:10.224Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356"}
{"level":"info","ts":"2026-09-18T10:13:10.224Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356","nodeType":"worker","rayStartParams":{"address":"long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:13:10.224Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356","rayStartCmd":"ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:10.224Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:10.224Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356","enabled":true}
{"level":"error","ts":"2026-09-18T10:13:10.230Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:10.230Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:13:10.230Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356"}
{"level":"error","ts":"2026-09-18T10:13:10.230Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"0741911e-1ad5-4df8-b425-7087d79ec356","error":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:10.240Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8"}
{"level":"info","ts":"2026-09-18T10:13:10.240Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8"}
{"level":"info","ts":"2026-09-18T10:13:10.240Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8"}
{"level":"info","ts":"2026-09-18T10:13:10.240Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:10.240Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8"}
{"level":"info","ts":"2026-09-18T10:13:10.240Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:13:10.240Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8"}
{"level":"info","ts":"2026-09-18T10:13:10.240Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","mode":"IntegratedOAuth","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:10.240Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","cluster":"long-running-5qfl8","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:13:10.240Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","cluster":"long-running-5qfl8","authType":"IntegratedOAuth","serviceAccount":"long-running-5qfl8-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T10:13:10.240Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-5qfl8","ray.io/group":"headgroup","ray.io/identifier":"long-running-5qfl8-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:13:10.240Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T10:13:10.240Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:13:10.240Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:13:10.240Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","enabled":true}
{"level":"error","ts":"2026-09-18T10:13:10.249Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-5qfl8-head-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:10.249Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","error":"FailedCreateHeadPod\npods \"long-running-5qfl8-head-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:13:10.250Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:10.250Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:10.250Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:10.250Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","name":"long-running-5qfl8","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:09Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.42","serviceIP":"10.133.0.42","podName":"long-running-5qfl8-head-kflfc","serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:13:09Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T10:13:09Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-5qfl8-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:10Z","stateTransitionTimes":{"ready":"2026-09-18T10:05:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5qfl8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:13:09Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:05:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:03:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:03:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T10:13:09Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"long-running-5qfl8-head-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:13:10.273Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8"}
{"level":"error","ts":"2026-09-18T10:13:10.273Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"dea6607e-0ccc-4467-8524-5d00c27fc7f8","error":"FailedCreateHeadPod\npods \"long-running-5qfl8-head-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:10.273Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"1259dd33-bc46-40ce-b74e-1d702d32d332","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:10.273Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"07d4aa9f-b722-43ce-89f5-988e7535074b","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-09-18T10:13:10.273Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"07d4aa9f-b722-43ce-89f5-988e7535074b","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-09-18 10:03:20 +0000 UTC","Now":"2026-09-18T10:13:10.273Z","ShutdownTime":"2026-09-18T10:13:20.000Z"}
{"level":"info","ts":"2026-09-18T10:13:10.273Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"07d4aa9f-b722-43ce-89f5-988e7535074b","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-09-18T10:13:20.000Z","requeueAfter":11.726236085}
{"level":"info","ts":"2026-09-18T10:13:10.273Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"53520f69-7197-4134-8fde-86e4f0924d0a","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:10.285Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"1259dd33-bc46-40ce-b74e-1d702d32d332","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:10.312Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"53520f69-7197-4134-8fde-86e4f0924d0a","certificate":"ray-head-cert-long-running-5qfl8","dnsNames":["long-running-5qfl8-head-svc","localhost","long-running-5qfl8-head-svc.test-ns-ltx56.svc","long-running-5qfl8-head-svc.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:13:10.339Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"53520f69-7197-4134-8fde-86e4f0924d0a","certificate":"ray-worker-cert-long-running-5qfl8","dnsNames":["long-running-5qfl8-worker-svc","localhost","long-running-5qfl8-small-group","long-running-5qfl8-worker-svc.test-ns-ltx56.svc","long-running-5qfl8-worker-svc.test-ns-ltx56.svc.cluster.local","*.long-running-5qfl8-worker-svc.test-ns-ltx56.svc","*.long-running-5qfl8-worker-svc.test-ns-ltx56.svc.cluster.local","*-worker-*.test-ns-ltx56.svc","*-worker-*.test-ns-ltx56.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:13:10.339Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"53520f69-7197-4134-8fde-86e4f0924d0a"}
{"level":"info","ts":"2026-09-18T10:13:11.272Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840"}
{"level":"info","ts":"2026-09-18T10:13:11.272Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840"}
{"level":"info","ts":"2026-09-18T10:13:11.272Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840"}
{"level":"info","ts":"2026-09-18T10:13:11.272Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840","1 head service found":"long-running-5qfl8-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:11.272Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840"}
{"level":"info","ts":"2026-09-18T10:13:11.272Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:13:11.272Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840"}
{"level":"info","ts":"2026-09-18T10:13:11.272Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840","mode":"IntegratedOAuth","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:11.272Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840","cluster":"long-running-5qfl8","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:13:11.272Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840","cluster":"long-running-5qfl8","authType":"IntegratedOAuth","serviceAccount":"long-running-5qfl8-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T10:13:11.272Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-5qfl8","ray.io/group":"headgroup","ray.io/identifier":"long-running-5qfl8-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:13:11.272Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T10:13:11.272Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:13:11.272Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:13:11.272Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840","enabled":true}
{"level":"error","ts":"2026-09-18T10:13:11.277Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-5qfl8-head-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:11.277Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840","error":"FailedCreateHeadPod\npods \"long-running-5qfl8-head-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:13:11.277Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:11.277Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:11.277Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840","filter labels":["test-ns-ltx56",{"ray.io/cluster":"long-running-5qfl8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:13:11.277Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840"}
{"level":"error","ts":"2026-09-18T10:13:11.277Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"ee051bd0-ab2e-48ff-a31e-642c2619e840","error":"FailedCreateHeadPod\npods \"long-running-5qfl8-head-\" is forbidden: unable to create new content in namespace test-ns-ltx56 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:13.492Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","namespace":"test-ns-ltx56"},"reconcileID":"77d7ce8b-6708-418c-9a7c-8ea2d5bf80e0"}
{"level":"info","ts":"2026-09-18T10:13:13.501Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"8f68da3c-68a9-4651-9cee-fd5ee3dada43","DeletionTimestamp":"2026-09-18 10:13:13 +0000 UTC"}
{"level":"info","ts":"2026-09-18T10:13:13.501Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"8f68da3c-68a9-4651-9cee-fd5ee3dada43","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T10:13:13.522Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-ltx56"},"reconcileID":"4232f445-585a-4b34-84da-bd0292477a10"}
{"level":"info","ts":"2026-09-18T10:13:13.522Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"9a75559b-301b-445a-aac1-0576f3065f1c","DeletionTimestamp":"2026-09-18 10:13:13 +0000 UTC"}
{"level":"error","ts":"2026-09-18T10:13:13.522Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"9a75559b-301b-445a-aac1-0576f3065f1c","error":"Post \"http:///api/jobs/long-running-rxs4j/stop\": http: no Host in request URL","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:13.522Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"9a75559b-301b-445a-aac1-0576f3065f1c","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T10:13:13.551Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"d961b02f-e029-4ad3-8ce2-aefdae2569cd"}
{"level":"info","ts":"2026-09-18T10:13:13.551Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"2607bbd5-0f74-45e3-8912-1a1c18486d58","DeletionTimestamp":"2026-09-18 10:13:13 +0000 UTC"}
{"level":"info","ts":"2026-09-18T10:13:13.551Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"2607bbd5-0f74-45e3-8912-1a1c18486d58","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T10:13:13.570Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fbf56117-2b1d-4c68-840e-f9a8de63e160"}
{"level":"info","ts":"2026-09-18T10:13:13.570Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"ef96d724-4924-4604-b8c1-070d25b034d0"}
{"level":"info","ts":"2026-09-18T10:13:13.571Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a76dede6-42a4-4e75-ba19-e80f0f74b5bf","namespacedName":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:13:13.571Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a76dede6-42a4-4e75-ba19-e80f0f74b5bf","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:13.571Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a76dede6-42a4-4e75-ba19-e80f0f74b5bf","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:13.571Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"310b2c90-c7da-4171-9713-adaeb8771721","rayCluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:13.571Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a76dede6-42a4-4e75-ba19-e80f0f74b5bf","namespace":"test-ns-ltx56","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T10:13:13.571Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"310b2c90-c7da-4171-9713-adaeb8771721"}
{"level":"info","ts":"2026-09-18T10:13:13.574Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"a620db3a-bd59-44df-ada4-dfb00f87a7ce","DeletionTimestamp":"2026-09-18 10:13:13 +0000 UTC"}
{"level":"error","ts":"2026-09-18T10:13:13.574Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"a620db3a-bd59-44df-ada4-dfb00f87a7ce","error":"RayCluster.ray.io \"ttl-initializing-rmrfw\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:121\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:13:13.574Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"a620db3a-bd59-44df-ada4-dfb00f87a7ce","error":"Post \"http:///api/jobs/ttl-initializing-rcc8l/stop\": http: no Host in request URL","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:13.574Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"a620db3a-bd59-44df-ada4-dfb00f87a7ce","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T10:13:13.578Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a76dede6-42a4-4e75-ba19-e80f0f74b5bf","name":"kuberay-gateway-access","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:13:13.587Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a76dede6-42a4-4e75-ba19-e80f0f74b5bf","name":"test-ns-ltx56-long-running-5qfl8","namespace":"opendatahub"}
{"level":"info","ts":"2026-09-18T10:13:13.596Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a76dede6-42a4-4e75-ba19-e80f0f74b5bf","configMap":"kube-rbac-proxy-config-long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:13.596Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"980cf603-d4e7-4eba-b1e7-1602fcadf628"}
{"level":"info","ts":"2026-09-18T10:13:13.596Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9efe9eee-3aa3-4f8e-a1a2-a4f791b09a97","DeletionTimestamp":"2026-09-18 10:13:13 +0000 UTC"}
{"level":"error","ts":"2026-09-18T10:13:13.596Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9efe9eee-3aa3-4f8e-a1a2-a4f791b09a97","error":"RayCluster.ray.io \"ttl-waiting-4wsk7\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:121\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:13:13.600Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9efe9eee-3aa3-4f8e-a1a2-a4f791b09a97","error":"Post \"http://ttl-waiting-4wsk7-head-svc.test-ns-ltx56.svc.cluster.local:8265/api/jobs//stop\": dial tcp: lookup ttl-waiting-4wsk7-head-svc.test-ns-ltx56.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:13.600Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"9efe9eee-3aa3-4f8e-a1a2-a4f791b09a97","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-09-18T10:13:13.601Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a76dede6-42a4-4e75-ba19-e80f0f74b5bf","serviceAccount":"long-running-5qfl8-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:13:13.609Z","logger":"raycluster-resource","msg":"default","name":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:13.609Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-5qfl8","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:13:13.666Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"reconcileID":"fec94393-d737-4899-bbd0-abe12fb07f85"}
{"level":"info","ts":"2026-09-18T10:13:13.666Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a76dede6-42a4-4e75-ba19-e80f0f74b5bf","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:13.666Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"19ccefa1-452b-44bc-9ccd-08a9f688ba34","namespacedName":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:13:13.666Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"19ccefa1-452b-44bc-9ccd-08a9f688ba34","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:13.666Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"19ccefa1-452b-44bc-9ccd-08a9f688ba34","cluster":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:13.667Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"19ccefa1-452b-44bc-9ccd-08a9f688ba34","namespace":"test-ns-ltx56","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T10:13:13.667Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"19ccefa1-452b-44bc-9ccd-08a9f688ba34","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-09-18T10:13:13.667Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"39685be4-705a-4e0f-a57a-fed760123ed5","namespace":"test-ns-ltx56","clusterName":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:13.667Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"01201d12-63f6-4c68-a035-e40cc3183edd"}
{"level":"info","ts":"2026-09-18T10:13:13.667Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"87fe96cc-cf05-484b-b875-cb57e5896abd"}
{"level":"info","ts":"2026-09-18T10:13:13.667Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"7a06e3aa-6a07-4d3f-9801-044889060f01"}
{"level":"info","ts":"2026-09-18T10:13:13.667Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"f4681e26-bc1b-4b9b-a34d-3cc6a7696912"}
{"level":"info","ts":"2026-09-18T10:13:13.667Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"9b2821bf-34d5-4c7b-8bfe-9ba63e82b11e"}
{"level":"info","ts":"2026-09-18T10:13:13.667Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"75aed6d5-6277-4201-a242-65ce00bf848a"}
{"level":"error","ts":"2026-09-18T10:13:13.674Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"39685be4-705a-4e0f-a57a-fed760123ed5","certificate":"ray-head-cert-long-running-5qfl8","error":"certificates.cert-manager.io \"ray-head-cert-long-running-5qfl8\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:13:13.675Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"19ccefa1-452b-44bc-9ccd-08a9f688ba34","error":"rayclusters.ray.io \"long-running-5qfl8\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:196\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:137\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:13.675Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"19ccefa1-452b-44bc-9ccd-08a9f688ba34"}
{"level":"error","ts":"2026-09-18T10:13:13.675Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"19ccefa1-452b-44bc-9ccd-08a9f688ba34","error":"rayclusters.ray.io \"long-running-5qfl8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:13.675Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"b23a8078-e925-4c83-8611-5c51d8d1bffb","namespacedName":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:13:13.675Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"b23a8078-e925-4c83-8611-5c51d8d1bffb"}
{"level":"info","ts":"2026-09-18T10:13:13.675Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"b23a8078-e925-4c83-8611-5c51d8d1bffb","namespace":"test-ns-ltx56","authClusters":0}
{"level":"info","ts":"2026-09-18T10:13:13.675Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"b23a8078-e925-4c83-8611-5c51d8d1bffb","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:13:13.680Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"822c5f92-61fd-4e49-a2cd-0f7020a4f163","namespacedName":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"}}
{"level":"info","ts":"2026-09-18T10:13:13.680Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"822c5f92-61fd-4e49-a2cd-0f7020a4f163"}
{"level":"info","ts":"2026-09-18T10:13:13.680Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"822c5f92-61fd-4e49-a2cd-0f7020a4f163","namespace":"test-ns-ltx56","authClusters":0}
{"level":"info","ts":"2026-09-18T10:13:13.680Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"822c5f92-61fd-4e49-a2cd-0f7020a4f163","namespace":"test-ns-ltx56"}
{"level":"info","ts":"2026-09-18T10:13:13.701Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"39685be4-705a-4e0f-a57a-fed760123ed5","errorCount":1}
{"level":"info","ts":"2026-09-18T10:13:13.701Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"39685be4-705a-4e0f-a57a-fed760123ed5"}
{"level":"error","ts":"2026-09-18T10:13:13.701Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"39685be4-705a-4e0f-a57a-fed760123ed5","error":"certificates.cert-manager.io \"ray-head-cert-long-running-5qfl8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:13.706Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a1e932b8-1bd9-4751-b9d6-2df072791642","namespace":"test-ns-ltx56","clusterName":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:13.706Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5qfl8","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"long-running-5qfl8","reconcileID":"a1e932b8-1bd9-4751-b9d6-2df072791642","namespace":"test-ns-ltx56","clusterName":"long-running-5qfl8"}
{"level":"info","ts":"2026-09-18T10:13:14.199Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f9ae364e-8f17-43b5-acd7-8340eebb052e"}
{"level":"info","ts":"2026-09-18T10:13:14.199Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f9ae364e-8f17-43b5-acd7-8340eebb052e"}
{"level":"info","ts":"2026-09-18T10:13:14.199Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f9ae364e-8f17-43b5-acd7-8340eebb052e"}
{"level":"info","ts":"2026-09-18T10:13:14.199Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f9ae364e-8f17-43b5-acd7-8340eebb052e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:14.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f9ae364e-8f17-43b5-acd7-8340eebb052e","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:14.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f9ae364e-8f17-43b5-acd7-8340eebb052e","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:14.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f9ae364e-8f17-43b5-acd7-8340eebb052e","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:13:14.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f9ae364e-8f17-43b5-acd7-8340eebb052e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:14.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f9ae364e-8f17-43b5-acd7-8340eebb052e","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:14.200Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f9ae364e-8f17-43b5-acd7-8340eebb052e","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:14.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f9ae364e-8f17-43b5-acd7-8340eebb052e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:14.200Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f9ae364e-8f17-43b5-acd7-8340eebb052e","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:14.200Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f9ae364e-8f17-43b5-acd7-8340eebb052e","podName":"ray-cluster-head-w667z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:14.201Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f9ae364e-8f17-43b5-acd7-8340eebb052e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:14.201Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f9ae364e-8f17-43b5-acd7-8340eebb052e","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:15.212Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a3dd715d-9adb-4142-9f95-94e298d0365e"}
{"level":"info","ts":"2026-09-18T10:13:15.212Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a3dd715d-9adb-4142-9f95-94e298d0365e"}
{"level":"info","ts":"2026-09-18T10:13:15.212Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a3dd715d-9adb-4142-9f95-94e298d0365e"}
{"level":"info","ts":"2026-09-18T10:13:15.212Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a3dd715d-9adb-4142-9f95-94e298d0365e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:15.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a3dd715d-9adb-4142-9f95-94e298d0365e","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:15.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a3dd715d-9adb-4142-9f95-94e298d0365e","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:15.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a3dd715d-9adb-4142-9f95-94e298d0365e","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:13:15.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a3dd715d-9adb-4142-9f95-94e298d0365e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:15.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a3dd715d-9adb-4142-9f95-94e298d0365e","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:15.212Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a3dd715d-9adb-4142-9f95-94e298d0365e","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:15.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a3dd715d-9adb-4142-9f95-94e298d0365e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:15.212Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a3dd715d-9adb-4142-9f95-94e298d0365e","podName":"ray-cluster-head-w667z","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:13:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:13:15.212Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a3dd715d-9adb-4142-9f95-94e298d0365e","podName":"ray-cluster-head-w667z","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:13:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:13:15.213Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a3dd715d-9adb-4142-9f95-94e298d0365e","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]; autoscaler: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:13:15.232Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"2c24489c-04a6-4448-8b7b-18f79764ccbb","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:15.232Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"2c24489c-04a6-4448-8b7b-18f79764ccbb","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:13:15.232Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"2c24489c-04a6-4448-8b7b-18f79764ccbb","rayCluster":"ray-cluster","podIPs":["10.132.0.59"]}
{"level":"info","ts":"2026-09-18T10:13:15.232Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"bf8ac5ff-70b6-40c1-ba7a-ba248c9c227a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:15.251Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"bf8ac5ff-70b6-40c1-ba7a-ba248c9c227a","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:13:15.271Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"bf8ac5ff-70b6-40c1-ba7a-ba248c9c227a","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:13:15.271Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"bf8ac5ff-70b6-40c1-ba7a-ba248c9c227a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:16.523Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter-auth","namespace":"test-ns-ltx56"},"reconcileID":"267aa546-32d1-43fc-a877-cee994f3f214"}
{"level":"info","ts":"2026-09-18T10:13:16.552Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"long-running","namespace":"test-ns-ltx56"},"reconcileID":"a9c0214e-db3e-4fc9-943d-5694b06cf8cb"}
{"level":"info","ts":"2026-09-18T10:13:16.574Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-ltx56"},"reconcileID":"0bd34d70-756b-4f69-b066-c7419fe175ec"}
{"level":"info","ts":"2026-09-18T10:13:16.597Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-ltx56"},"reconcileID":"f9d0938a-9a86-4549-9cb0-e5ed2baaa4b1"}
{"level":"info","ts":"2026-09-18T10:13:16.668Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-ltx56"},"reconcileID":"62b4eaf9-4d4f-4822-a2aa-fb15ab290354"}
{"level":"info","ts":"2026-09-18T10:13:17.232Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fe12ce6e-6c3f-45ea-8a7d-a17863d42422"}
{"level":"info","ts":"2026-09-18T10:13:17.232Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fe12ce6e-6c3f-45ea-8a7d-a17863d42422"}
{"level":"info","ts":"2026-09-18T10:13:17.232Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fe12ce6e-6c3f-45ea-8a7d-a17863d42422"}
{"level":"info","ts":"2026-09-18T10:13:17.232Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fe12ce6e-6c3f-45ea-8a7d-a17863d42422","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:17.232Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fe12ce6e-6c3f-45ea-8a7d-a17863d42422","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:17.232Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fe12ce6e-6c3f-45ea-8a7d-a17863d42422","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:17.232Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fe12ce6e-6c3f-45ea-8a7d-a17863d42422","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:13:17.232Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fe12ce6e-6c3f-45ea-8a7d-a17863d42422","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:17.232Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fe12ce6e-6c3f-45ea-8a7d-a17863d42422","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:17.232Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fe12ce6e-6c3f-45ea-8a7d-a17863d42422","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:17.232Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fe12ce6e-6c3f-45ea-8a7d-a17863d42422","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:17.232Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fe12ce6e-6c3f-45ea-8a7d-a17863d42422","podName":"ray-cluster-head-w667z","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:13:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:13:17.232Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fe12ce6e-6c3f-45ea-8a7d-a17863d42422","podName":"ray-cluster-head-w667z","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:13:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:13:17.233Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fe12ce6e-6c3f-45ea-8a7d-a17863d42422","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:17.233Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fe12ce6e-6c3f-45ea-8a7d-a17863d42422","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:35.266Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"5c842a4c-e601-461b-846f-0b621ef3e26a","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:35.266Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"5c842a4c-e601-461b-846f-0b621ef3e26a","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:13:35.266Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"5c842a4c-e601-461b-846f-0b621ef3e26a","rayCluster":"ray-cluster","podIPs":["10.132.0.59"]}
{"level":"info","ts":"2026-09-18T10:13:36.241Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"143a8651-8311-4195-ad86-435665879fd2"}
{"level":"info","ts":"2026-09-18T10:13:36.241Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"143a8651-8311-4195-ad86-435665879fd2"}
{"level":"info","ts":"2026-09-18T10:13:36.241Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"143a8651-8311-4195-ad86-435665879fd2"}
{"level":"info","ts":"2026-09-18T10:13:36.242Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"143a8651-8311-4195-ad86-435665879fd2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:36.242Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"143a8651-8311-4195-ad86-435665879fd2","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:36.242Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"143a8651-8311-4195-ad86-435665879fd2","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:36.242Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"143a8651-8311-4195-ad86-435665879fd2","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:13:36.242Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"143a8651-8311-4195-ad86-435665879fd2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:36.242Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"143a8651-8311-4195-ad86-435665879fd2","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:36.242Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"143a8651-8311-4195-ad86-435665879fd2","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:36.242Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"143a8651-8311-4195-ad86-435665879fd2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:36.242Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"143a8651-8311-4195-ad86-435665879fd2","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:36Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:13:36.257Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1fdc7f39-fee6-4897-ac8b-99423d4d837a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:36.257Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"ffd5a74c-e8e3-4d00-9c96-1056775f4175","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:36.257Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"ffd5a74c-e8e3-4d00-9c96-1056775f4175","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:13:36.257Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"ffd5a74c-e8e3-4d00-9c96-1056775f4175","rayCluster":"ray-cluster","podIPs":["10.132.0.59"]}
{"level":"info","ts":"2026-09-18T10:13:36.279Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1fdc7f39-fee6-4897-ac8b-99423d4d837a","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:13:36.291Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1fdc7f39-fee6-4897-ac8b-99423d4d837a","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:13:36.291Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1fdc7f39-fee6-4897-ac8b-99423d4d837a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:38.258Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"edce3642-b3b8-4f5c-a3bf-2b909256919c"}
{"level":"info","ts":"2026-09-18T10:13:38.258Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"edce3642-b3b8-4f5c-a3bf-2b909256919c"}
{"level":"info","ts":"2026-09-18T10:13:38.258Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"edce3642-b3b8-4f5c-a3bf-2b909256919c"}
{"level":"info","ts":"2026-09-18T10:13:38.258Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"edce3642-b3b8-4f5c-a3bf-2b909256919c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:38.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"edce3642-b3b8-4f5c-a3bf-2b909256919c","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:38.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"edce3642-b3b8-4f5c-a3bf-2b909256919c","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:38.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"edce3642-b3b8-4f5c-a3bf-2b909256919c","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:13:38.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"edce3642-b3b8-4f5c-a3bf-2b909256919c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:38.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"edce3642-b3b8-4f5c-a3bf-2b909256919c","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:38.258Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"edce3642-b3b8-4f5c-a3bf-2b909256919c","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:38.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"edce3642-b3b8-4f5c-a3bf-2b909256919c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:38.258Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"edce3642-b3b8-4f5c-a3bf-2b909256919c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:38.258Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"edce3642-b3b8-4f5c-a3bf-2b909256919c","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:42.441Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:42.441Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-pmtv8"}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e1322834-d504-4a55-844f-7a2d3a6d445d","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pmtv8"}}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e1322834-d504-4a55-844f-7a2d3a6d445d","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e1322834-d504-4a55-844f-7a2d3a6d445d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6"}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6"}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6"}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6"}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","nodeType":"worker","rayStartParams":{"address":"ray-cluster-head-svc.test-ns-pmtv8.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"1G"},"requests":{"cpu":"1","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","rayStartCmd":"ray start  --address=ray-cluster-head-svc.test-ns-pmtv8.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=ray-cluster-head-svc.test-ns-pmtv8.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","enabled":true}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"669602a4-c8e4-4dbc-a71c-b1ca555cbe96","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:42.458Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"669602a4-c8e4-4dbc-a71c-b1ca555cbe96","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:13:42.459Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"669602a4-c8e4-4dbc-a71c-b1ca555cbe96","rayCluster":"ray-cluster","podIPs":["10.132.0.59"]}
{"level":"info","ts":"2026-09-18T10:13:42.459Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b778d4b9-2183-4517-9e25-7a9d0e814a74","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:42.491Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b778d4b9-2183-4517-9e25-7a9d0e814a74","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:13:42.492Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e1322834-d504-4a55-844f-7a2d3a6d445d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:42.494Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"5883a9e0-5865-4bc8-b3f6-5968e23d758a","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:42.494Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"5883a9e0-5865-4bc8-b3f6-5968e23d758a","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:13:42.494Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"5883a9e0-5865-4bc8-b3f6-5968e23d758a","rayCluster":"ray-cluster","podIPs":["10.132.0.59"]}
{"level":"info","ts":"2026-09-18T10:13:42.511Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","name":"ray-cluster-small-group-worker-49ccd"}
{"level":"info","ts":"2026-09-18T10:13:42.511Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:42.511Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:36Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:42Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-09-18T10:13:42.518Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b778d4b9-2183-4517-9e25-7a9d0e814a74","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:13:42.518Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b778d4b9-2183-4517-9e25-7a9d0e814a74","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:42.519Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"fb06e2d1-e7d1-45d3-8d24-1f45c0dedcf9","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pmtv8","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6"}
{"level":"error","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"06c5bf5a-90b1-460b-9b02-13669af918a6","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ad33b5d9-8de1-4917-a71d-2e78c8949912"}
{"level":"info","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ad33b5d9-8de1-4917-a71d-2e78c8949912"}
{"level":"info","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ad33b5d9-8de1-4917-a71d-2e78c8949912"}
{"level":"info","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ad33b5d9-8de1-4917-a71d-2e78c8949912","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ad33b5d9-8de1-4917-a71d-2e78c8949912","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ad33b5d9-8de1-4917-a71d-2e78c8949912","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ad33b5d9-8de1-4917-a71d-2e78c8949912","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ad33b5d9-8de1-4917-a71d-2e78c8949912","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ad33b5d9-8de1-4917-a71d-2e78c8949912","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ad33b5d9-8de1-4917-a71d-2e78c8949912","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ad33b5d9-8de1-4917-a71d-2e78c8949912","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ad33b5d9-8de1-4917-a71d-2e78c8949912","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ad33b5d9-8de1-4917-a71d-2e78c8949912","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:42.522Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ad33b5d9-8de1-4917-a71d-2e78c8949912","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:36Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:42Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-09-18T10:13:42.547Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"fb06e2d1-e7d1-45d3-8d24-1f45c0dedcf9","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:13:42.554Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"28af6ff4-0c7d-490e-b9a7-d16991c9370a","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:42.554Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"28af6ff4-0c7d-490e-b9a7-d16991c9370a","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:13:42.554Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"28af6ff4-0c7d-490e-b9a7-d16991c9370a","rayCluster":"ray-cluster","podIPs":["10.132.0.59"]}
{"level":"info","ts":"2026-09-18T10:13:42.554Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"12bed315-8195-4ee9-98a3-245bc4432777"}
{"level":"info","ts":"2026-09-18T10:13:42.554Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"12bed315-8195-4ee9-98a3-245bc4432777"}
{"level":"info","ts":"2026-09-18T10:13:42.554Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"12bed315-8195-4ee9-98a3-245bc4432777"}
{"level":"info","ts":"2026-09-18T10:13:42.554Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"12bed315-8195-4ee9-98a3-245bc4432777","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:42.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"12bed315-8195-4ee9-98a3-245bc4432777","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:42.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"12bed315-8195-4ee9-98a3-245bc4432777","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:42.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"12bed315-8195-4ee9-98a3-245bc4432777","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:42.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"12bed315-8195-4ee9-98a3-245bc4432777","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:42.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"12bed315-8195-4ee9-98a3-245bc4432777","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:42.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"12bed315-8195-4ee9-98a3-245bc4432777","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:42.555Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"12bed315-8195-4ee9-98a3-245bc4432777","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:13:42.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"12bed315-8195-4ee9-98a3-245bc4432777","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:42.555Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"12bed315-8195-4ee9-98a3-245bc4432777","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:42.555Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"12bed315-8195-4ee9-98a3-245bc4432777","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:42.555Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"12bed315-8195-4ee9-98a3-245bc4432777","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:42.559Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a4074843-58ff-4722-85c0-9364c9ff8368"}
{"level":"info","ts":"2026-09-18T10:13:42.565Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a4074843-58ff-4722-85c0-9364c9ff8368"}
{"level":"info","ts":"2026-09-18T10:13:42.565Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a4074843-58ff-4722-85c0-9364c9ff8368"}
{"level":"info","ts":"2026-09-18T10:13:42.566Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a4074843-58ff-4722-85c0-9364c9ff8368","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:42.566Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a4074843-58ff-4722-85c0-9364c9ff8368","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:42.566Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a4074843-58ff-4722-85c0-9364c9ff8368","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:42.566Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a4074843-58ff-4722-85c0-9364c9ff8368","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:42.566Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a4074843-58ff-4722-85c0-9364c9ff8368","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:42.566Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a4074843-58ff-4722-85c0-9364c9ff8368","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:42.566Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a4074843-58ff-4722-85c0-9364c9ff8368","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:42.566Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a4074843-58ff-4722-85c0-9364c9ff8368","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:13:42.566Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a4074843-58ff-4722-85c0-9364c9ff8368","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:42.566Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a4074843-58ff-4722-85c0-9364c9ff8368","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:42.566Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a4074843-58ff-4722-85c0-9364c9ff8368","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:42.566Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a4074843-58ff-4722-85c0-9364c9ff8368","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:42.577Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"fb06e2d1-e7d1-45d3-8d24-1f45c0dedcf9","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:13:42.577Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"fb06e2d1-e7d1-45d3-8d24-1f45c0dedcf9","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:42.577Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e53eb9c9-5ae0-4951-a1fa-68f280796b06","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:42.601Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e53eb9c9-5ae0-4951-a1fa-68f280796b06","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:13:42.608Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8059eb6b-b9b3-4e64-aae3-1093fe12273d"}
{"level":"info","ts":"2026-09-18T10:13:42.608Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8059eb6b-b9b3-4e64-aae3-1093fe12273d"}
{"level":"info","ts":"2026-09-18T10:13:42.608Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8059eb6b-b9b3-4e64-aae3-1093fe12273d"}
{"level":"info","ts":"2026-09-18T10:13:42.608Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8059eb6b-b9b3-4e64-aae3-1093fe12273d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:42.608Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8059eb6b-b9b3-4e64-aae3-1093fe12273d","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:42.608Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8059eb6b-b9b3-4e64-aae3-1093fe12273d","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:42.608Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8059eb6b-b9b3-4e64-aae3-1093fe12273d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:42.608Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8059eb6b-b9b3-4e64-aae3-1093fe12273d","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:42.608Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8059eb6b-b9b3-4e64-aae3-1093fe12273d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:42.608Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8059eb6b-b9b3-4e64-aae3-1093fe12273d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:42.608Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8059eb6b-b9b3-4e64-aae3-1093fe12273d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:13:42.608Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8059eb6b-b9b3-4e64-aae3-1093fe12273d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:42.608Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8059eb6b-b9b3-4e64-aae3-1093fe12273d","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:42.608Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8059eb6b-b9b3-4e64-aae3-1093fe12273d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:42.608Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8059eb6b-b9b3-4e64-aae3-1093fe12273d","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:42.625Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e53eb9c9-5ae0-4951-a1fa-68f280796b06","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:13:42.625Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e53eb9c9-5ae0-4951-a1fa-68f280796b06","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:43.142Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"35d31bba-140f-4cd4-8585-2538d59b7025"}
{"level":"info","ts":"2026-09-18T10:13:43.142Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"35d31bba-140f-4cd4-8585-2538d59b7025"}
{"level":"info","ts":"2026-09-18T10:13:43.142Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"35d31bba-140f-4cd4-8585-2538d59b7025"}
{"level":"info","ts":"2026-09-18T10:13:43.142Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"35d31bba-140f-4cd4-8585-2538d59b7025","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:43.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"35d31bba-140f-4cd4-8585-2538d59b7025","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:43.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"35d31bba-140f-4cd4-8585-2538d59b7025","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:43.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"35d31bba-140f-4cd4-8585-2538d59b7025","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:43.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"35d31bba-140f-4cd4-8585-2538d59b7025","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:43.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"35d31bba-140f-4cd4-8585-2538d59b7025","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:43.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"35d31bba-140f-4cd4-8585-2538d59b7025","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:43.142Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"35d31bba-140f-4cd4-8585-2538d59b7025","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:13:43.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"35d31bba-140f-4cd4-8585-2538d59b7025","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:43.142Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"35d31bba-140f-4cd4-8585-2538d59b7025","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:43.142Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"35d31bba-140f-4cd4-8585-2538d59b7025","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:43.142Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"35d31bba-140f-4cd4-8585-2538d59b7025","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:43.351Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5d436616-9cf2-4f78-82e2-5efb578e7720"}
{"level":"info","ts":"2026-09-18T10:13:43.351Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5d436616-9cf2-4f78-82e2-5efb578e7720"}
{"level":"info","ts":"2026-09-18T10:13:43.351Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5d436616-9cf2-4f78-82e2-5efb578e7720"}
{"level":"info","ts":"2026-09-18T10:13:43.351Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5d436616-9cf2-4f78-82e2-5efb578e7720","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:43.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5d436616-9cf2-4f78-82e2-5efb578e7720","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:43.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5d436616-9cf2-4f78-82e2-5efb578e7720","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:43.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5d436616-9cf2-4f78-82e2-5efb578e7720","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:43.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5d436616-9cf2-4f78-82e2-5efb578e7720","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:43.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5d436616-9cf2-4f78-82e2-5efb578e7720","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:43.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5d436616-9cf2-4f78-82e2-5efb578e7720","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:43.351Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5d436616-9cf2-4f78-82e2-5efb578e7720","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:13:43.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5d436616-9cf2-4f78-82e2-5efb578e7720","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:43.351Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5d436616-9cf2-4f78-82e2-5efb578e7720","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:43.352Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5d436616-9cf2-4f78-82e2-5efb578e7720","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:43.352Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5d436616-9cf2-4f78-82e2-5efb578e7720","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:44.555Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5e6b6d8a-a99a-4cdb-b1d9-fb66be4db461"}
{"level":"info","ts":"2026-09-18T10:13:44.555Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5e6b6d8a-a99a-4cdb-b1d9-fb66be4db461"}
{"level":"info","ts":"2026-09-18T10:13:44.555Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5e6b6d8a-a99a-4cdb-b1d9-fb66be4db461"}
{"level":"info","ts":"2026-09-18T10:13:44.555Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5e6b6d8a-a99a-4cdb-b1d9-fb66be4db461","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:44.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5e6b6d8a-a99a-4cdb-b1d9-fb66be4db461","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:44.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5e6b6d8a-a99a-4cdb-b1d9-fb66be4db461","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:44.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5e6b6d8a-a99a-4cdb-b1d9-fb66be4db461","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:13:44.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5e6b6d8a-a99a-4cdb-b1d9-fb66be4db461","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:44.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5e6b6d8a-a99a-4cdb-b1d9-fb66be4db461","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:44.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5e6b6d8a-a99a-4cdb-b1d9-fb66be4db461","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:44.555Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5e6b6d8a-a99a-4cdb-b1d9-fb66be4db461","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:13:44.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5e6b6d8a-a99a-4cdb-b1d9-fb66be4db461","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:44.556Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5e6b6d8a-a99a-4cdb-b1d9-fb66be4db461","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:44.556Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5e6b6d8a-a99a-4cdb-b1d9-fb66be4db461","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:44.556Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"5e6b6d8a-a99a-4cdb-b1d9-fb66be4db461","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:47.580Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:47.580Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-pmtv8"}
{"level":"info","ts":"2026-09-18T10:13:47.606Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"525d6555-fbfa-4c10-b19d-74b6c9e83288","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:47.606Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"be3a0e7a-bd42-4c5a-a4ae-990c217ca276","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:47.606Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"525d6555-fbfa-4c10-b19d-74b6c9e83288","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:13:47.606Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"525d6555-fbfa-4c10-b19d-74b6c9e83288","pod":"ray-cluster-small-group-worker-49ccd","ip":"10.133.0.51"}
{"level":"info","ts":"2026-09-18T10:13:47.606Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b"}
{"level":"info","ts":"2026-09-18T10:13:47.606Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b"}
{"level":"info","ts":"2026-09-18T10:13:47.606Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b"}
{"level":"info","ts":"2026-09-18T10:13:47.606Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"83d85aed-4e52-461a-8d7f-24fad1755377","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pmtv8"}}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"83d85aed-4e52-461a-8d7f-24fad1755377","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"83d85aed-4e52-461a-8d7f-24fad1755377","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","workerReplicas":2,"NumOfHosts":1,"runningPods":1,"diff":1}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":1}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b"}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","nodeType":"worker","rayStartParams":{"address":"ray-cluster-head-svc.test-ns-pmtv8.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"1G"},"requests":{"cpu":"1","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","rayStartCmd":"ray start  --address=ray-cluster-head-svc.test-ns-pmtv8.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=ray-cluster-head-svc.test-ns-pmtv8.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:13:47.607Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","enabled":true}
{"level":"info","ts":"2026-09-18T10:13:47.632Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"be3a0e7a-bd42-4c5a-a4ae-990c217ca276","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:13:47.640Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","name":"ray-cluster-small-group-worker-zfpb5"}
{"level":"info","ts":"2026-09-18T10:13:47.640Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:42Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:47Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-09-18T10:13:47.645Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"525d6555-fbfa-4c10-b19d-74b6c9e83288","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-pmtv8.svc","ray-cluster-head-svc.test-ns-pmtv8.svc.cluster.local"],"ipAddresses":["10.132.0.59","10.133.0.51","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:13:47.645Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"83d85aed-4e52-461a-8d7f-24fad1755377","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pmtv8","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b"}
{"level":"error","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ee1bff66-f1ad-4ea1-994a-3c112c18846b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6485d2ae-0125-4e0f-b6ce-03918549d684"}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6485d2ae-0125-4e0f-b6ce-03918549d684"}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6485d2ae-0125-4e0f-b6ce-03918549d684"}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6485d2ae-0125-4e0f-b6ce-03918549d684","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6485d2ae-0125-4e0f-b6ce-03918549d684","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6485d2ae-0125-4e0f-b6ce-03918549d684","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6485d2ae-0125-4e0f-b6ce-03918549d684","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6485d2ae-0125-4e0f-b6ce-03918549d684","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6485d2ae-0125-4e0f-b6ce-03918549d684","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6485d2ae-0125-4e0f-b6ce-03918549d684","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6485d2ae-0125-4e0f-b6ce-03918549d684","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6485d2ae-0125-4e0f-b6ce-03918549d684","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6485d2ae-0125-4e0f-b6ce-03918549d684","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6485d2ae-0125-4e0f-b6ce-03918549d684","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:47.672Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6485d2ae-0125-4e0f-b6ce-03918549d684","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:42Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:47Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-09-18T10:13:47.683Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"be3a0e7a-bd42-4c5a-a4ae-990c217ca276","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:13:47.683Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"be3a0e7a-bd42-4c5a-a4ae-990c217ca276","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:47.683Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"27e68b68-0891-424e-8f97-d67ba5b2bc76","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:47.693Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423"}
{"level":"info","ts":"2026-09-18T10:13:47.693Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423"}
{"level":"info","ts":"2026-09-18T10:13:47.693Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423"}
{"level":"info","ts":"2026-09-18T10:13:47.693Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:47.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:47.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:47.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:13:47.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:47.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:47.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:47.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:47.693Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:13:47.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:47.693Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:47.693Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:42Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:47Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-09-18T10:13:47.714Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"525d6555-fbfa-4c10-b19d-74b6c9e83288","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-pmtv8.svc","ray-cluster-worker-svc.test-ns-pmtv8.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-pmtv8.svc","*.ray-cluster-worker-svc.test-ns-pmtv8.svc.cluster.local","*-worker-*.test-ns-pmtv8.svc","*-worker-*.test-ns-pmtv8.svc.cluster.local"],"ipAddresses":["10.132.0.59","10.133.0.51","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:13:47.714Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"525d6555-fbfa-4c10-b19d-74b6c9e83288"}
{"level":"info","ts":"2026-09-18T10:13:47.714Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"c88699b1-e9c3-442a-b512-0a17cab59649","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:47.715Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pmtv8","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:13:47.715Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"c88699b1-e9c3-442a-b512-0a17cab59649","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:13:47.715Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423"}
{"level":"info","ts":"2026-09-18T10:13:47.715Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"c88699b1-e9c3-442a-b512-0a17cab59649","pod":"ray-cluster-small-group-worker-49ccd","ip":"10.133.0.51"}
{"level":"error","ts":"2026-09-18T10:13:47.715Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bd14ff86-3c3d-46e9-ab6b-cff31c02b423","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:13:47.715Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"c88699b1-e9c3-442a-b512-0a17cab59649"}
{"level":"info","ts":"2026-09-18T10:13:47.720Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1da6e1cd-a839-4d89-8f3a-b43a30043551"}
{"level":"info","ts":"2026-09-18T10:13:47.720Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1da6e1cd-a839-4d89-8f3a-b43a30043551"}
{"level":"info","ts":"2026-09-18T10:13:47.720Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1da6e1cd-a839-4d89-8f3a-b43a30043551"}
{"level":"info","ts":"2026-09-18T10:13:47.720Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1da6e1cd-a839-4d89-8f3a-b43a30043551","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:47.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1da6e1cd-a839-4d89-8f3a-b43a30043551","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:47.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1da6e1cd-a839-4d89-8f3a-b43a30043551","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:47.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1da6e1cd-a839-4d89-8f3a-b43a30043551","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:13:47.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1da6e1cd-a839-4d89-8f3a-b43a30043551","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:47.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1da6e1cd-a839-4d89-8f3a-b43a30043551","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:47.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1da6e1cd-a839-4d89-8f3a-b43a30043551","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:47.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1da6e1cd-a839-4d89-8f3a-b43a30043551","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:47.720Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1da6e1cd-a839-4d89-8f3a-b43a30043551","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:13:47.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1da6e1cd-a839-4d89-8f3a-b43a30043551","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:47.720Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1da6e1cd-a839-4d89-8f3a-b43a30043551","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:47.720Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1da6e1cd-a839-4d89-8f3a-b43a30043551","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:47.720Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1da6e1cd-a839-4d89-8f3a-b43a30043551","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:47.724Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"27e68b68-0891-424e-8f97-d67ba5b2bc76","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:13:47.725Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"c0fadbbf-71b6-4ff5-82ae-ab0e5ecfccd1"}
{"level":"info","ts":"2026-09-18T10:13:47.725Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"c0fadbbf-71b6-4ff5-82ae-ab0e5ecfccd1"}
{"level":"info","ts":"2026-09-18T10:13:47.725Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"c0fadbbf-71b6-4ff5-82ae-ab0e5ecfccd1"}
{"level":"info","ts":"2026-09-18T10:13:47.725Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"c0fadbbf-71b6-4ff5-82ae-ab0e5ecfccd1","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:47.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"c0fadbbf-71b6-4ff5-82ae-ab0e5ecfccd1","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:47.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"c0fadbbf-71b6-4ff5-82ae-ab0e5ecfccd1","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:47.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"c0fadbbf-71b6-4ff5-82ae-ab0e5ecfccd1","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:13:47.766Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"27e68b68-0891-424e-8f97-d67ba5b2bc76","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:13:47.766Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"27e68b68-0891-424e-8f97-d67ba5b2bc76","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:47.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"c0fadbbf-71b6-4ff5-82ae-ab0e5ecfccd1","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:47.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"c0fadbbf-71b6-4ff5-82ae-ab0e5ecfccd1","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:47.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"c0fadbbf-71b6-4ff5-82ae-ab0e5ecfccd1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:47.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"c0fadbbf-71b6-4ff5-82ae-ab0e5ecfccd1","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:47.766Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"c0fadbbf-71b6-4ff5-82ae-ab0e5ecfccd1","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:13:47.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"c0fadbbf-71b6-4ff5-82ae-ab0e5ecfccd1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:47.766Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"dd2151da-36e3-4f9b-9756-f0604ff93057","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:47.766Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"c0fadbbf-71b6-4ff5-82ae-ab0e5ecfccd1","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:47.766Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"c0fadbbf-71b6-4ff5-82ae-ab0e5ecfccd1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:47.766Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"c0fadbbf-71b6-4ff5-82ae-ab0e5ecfccd1","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:47.808Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"dd2151da-36e3-4f9b-9756-f0604ff93057","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:13:47.854Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"dd2151da-36e3-4f9b-9756-f0604ff93057","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:13:47.854Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"dd2151da-36e3-4f9b-9756-f0604ff93057","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:13:48.261Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"db476ba3-38df-4fde-8fc3-8565d5c214fe"}
{"level":"info","ts":"2026-09-18T10:13:48.261Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"db476ba3-38df-4fde-8fc3-8565d5c214fe"}
{"level":"info","ts":"2026-09-18T10:13:48.261Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"db476ba3-38df-4fde-8fc3-8565d5c214fe"}
{"level":"info","ts":"2026-09-18T10:13:48.261Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"db476ba3-38df-4fde-8fc3-8565d5c214fe","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:48.261Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"db476ba3-38df-4fde-8fc3-8565d5c214fe","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:48.261Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"db476ba3-38df-4fde-8fc3-8565d5c214fe","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:48.261Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"db476ba3-38df-4fde-8fc3-8565d5c214fe","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:13:48.261Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"db476ba3-38df-4fde-8fc3-8565d5c214fe","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:48.261Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"db476ba3-38df-4fde-8fc3-8565d5c214fe","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:48.261Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"db476ba3-38df-4fde-8fc3-8565d5c214fe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:48.261Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"db476ba3-38df-4fde-8fc3-8565d5c214fe","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:48.261Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"db476ba3-38df-4fde-8fc3-8565d5c214fe","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:13:48.261Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"db476ba3-38df-4fde-8fc3-8565d5c214fe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:48.262Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"db476ba3-38df-4fde-8fc3-8565d5c214fe","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:48.262Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"db476ba3-38df-4fde-8fc3-8565d5c214fe","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:48.262Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"db476ba3-38df-4fde-8fc3-8565d5c214fe","seconds":300}
{"level":"info","ts":"2026-09-18T10:13:48.366Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f895c648-c30d-4c8c-9d0d-d90d2131fbbd"}
{"level":"info","ts":"2026-09-18T10:13:48.366Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f895c648-c30d-4c8c-9d0d-d90d2131fbbd"}
{"level":"info","ts":"2026-09-18T10:13:48.366Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f895c648-c30d-4c8c-9d0d-d90d2131fbbd"}
{"level":"info","ts":"2026-09-18T10:13:48.366Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f895c648-c30d-4c8c-9d0d-d90d2131fbbd","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:13:48.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f895c648-c30d-4c8c-9d0d-d90d2131fbbd","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:13:48.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f895c648-c30d-4c8c-9d0d-d90d2131fbbd","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:48.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f895c648-c30d-4c8c-9d0d-d90d2131fbbd","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:13:48.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f895c648-c30d-4c8c-9d0d-d90d2131fbbd","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:48.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f895c648-c30d-4c8c-9d0d-d90d2131fbbd","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:13:48.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f895c648-c30d-4c8c-9d0d-d90d2131fbbd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:48.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f895c648-c30d-4c8c-9d0d-d90d2131fbbd","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:13:48.366Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f895c648-c30d-4c8c-9d0d-d90d2131fbbd","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:13:48.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f895c648-c30d-4c8c-9d0d-d90d2131fbbd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:13:48.367Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f895c648-c30d-4c8c-9d0d-d90d2131fbbd","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:13:48.367Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f895c648-c30d-4c8c-9d0d-d90d2131fbbd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:13:48.367Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"f895c648-c30d-4c8c-9d0d-d90d2131fbbd","seconds":300}
{"level":"info","ts":"2026-09-18T10:14:00.529Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"1eeaf276-5715-409a-95ab-30e08ea5cf98","namespace":"test-ns-ltx56","clusterName":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:14:00.529Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-4wsk7","namespace":"test-ns-ltx56"},"namespace":"test-ns-ltx56","name":"ttl-waiting-4wsk7","reconcileID":"1eeaf276-5715-409a-95ab-30e08ea5cf98","namespace":"test-ns-ltx56","clusterName":"ttl-waiting-4wsk7"}
{"level":"info","ts":"2026-09-18T10:14:17.715Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1c405bcc-8940-4011-8a04-49f97db907ef","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:14:17.715Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1c405bcc-8940-4011-8a04-49f97db907ef","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:14:17.715Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1c405bcc-8940-4011-8a04-49f97db907ef","pod":"ray-cluster-small-group-worker-49ccd","ip":"10.133.0.51"}
{"level":"info","ts":"2026-09-18T10:14:17.715Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1c405bcc-8940-4011-8a04-49f97db907ef","pod":"ray-cluster-small-group-worker-zfpb5","ip":"10.133.0.52"}
{"level":"info","ts":"2026-09-18T10:14:17.734Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1c405bcc-8940-4011-8a04-49f97db907ef","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-pmtv8.svc","ray-cluster-head-svc.test-ns-pmtv8.svc.cluster.local"],"ipAddresses":["10.132.0.59","10.133.0.51","10.133.0.52","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:14:17.753Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1c405bcc-8940-4011-8a04-49f97db907ef","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-pmtv8.svc","ray-cluster-worker-svc.test-ns-pmtv8.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-pmtv8.svc","*.ray-cluster-worker-svc.test-ns-pmtv8.svc.cluster.local","*-worker-*.test-ns-pmtv8.svc","*-worker-*.test-ns-pmtv8.svc.cluster.local"],"ipAddresses":["10.132.0.59","10.133.0.51","10.133.0.52","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:14:17.753Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1c405bcc-8940-4011-8a04-49f97db907ef","rayCluster":"ray-cluster","podIPs":["10.132.0.59","10.133.0.51","10.133.0.52"]}
{"level":"info","ts":"2026-09-18T10:15:03.684Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2638f5fb-687a-4a26-bcb6-971e03d6f1fb"}
{"level":"info","ts":"2026-09-18T10:15:03.684Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2638f5fb-687a-4a26-bcb6-971e03d6f1fb"}
{"level":"info","ts":"2026-09-18T10:15:03.684Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2638f5fb-687a-4a26-bcb6-971e03d6f1fb"}
{"level":"info","ts":"2026-09-18T10:15:03.684Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2638f5fb-687a-4a26-bcb6-971e03d6f1fb","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:15:03.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2638f5fb-687a-4a26-bcb6-971e03d6f1fb","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:15:03.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2638f5fb-687a-4a26-bcb6-971e03d6f1fb","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:03.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2638f5fb-687a-4a26-bcb6-971e03d6f1fb","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:15:03.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2638f5fb-687a-4a26-bcb6-971e03d6f1fb","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:03.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2638f5fb-687a-4a26-bcb6-971e03d6f1fb","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:03.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2638f5fb-687a-4a26-bcb6-971e03d6f1fb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:03.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2638f5fb-687a-4a26-bcb6-971e03d6f1fb","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:15:03.684Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2638f5fb-687a-4a26-bcb6-971e03d6f1fb","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:15:03.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2638f5fb-687a-4a26-bcb6-971e03d6f1fb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:03.685Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2638f5fb-687a-4a26-bcb6-971e03d6f1fb","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:15:03.685Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2638f5fb-687a-4a26-bcb6-971e03d6f1fb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:15:03.685Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2638f5fb-687a-4a26-bcb6-971e03d6f1fb","seconds":300}
{"level":"info","ts":"2026-09-18T10:15:04.677Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b7638d86-d208-446b-aee7-7056b45685d7"}
{"level":"info","ts":"2026-09-18T10:15:04.677Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b7638d86-d208-446b-aee7-7056b45685d7"}
{"level":"info","ts":"2026-09-18T10:15:04.677Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b7638d86-d208-446b-aee7-7056b45685d7"}
{"level":"info","ts":"2026-09-18T10:15:04.677Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b7638d86-d208-446b-aee7-7056b45685d7","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:15:04.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b7638d86-d208-446b-aee7-7056b45685d7","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:15:04.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b7638d86-d208-446b-aee7-7056b45685d7","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:04.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b7638d86-d208-446b-aee7-7056b45685d7","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:15:04.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b7638d86-d208-446b-aee7-7056b45685d7","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:04.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b7638d86-d208-446b-aee7-7056b45685d7","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:04.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b7638d86-d208-446b-aee7-7056b45685d7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:04.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b7638d86-d208-446b-aee7-7056b45685d7","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:15:04.677Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b7638d86-d208-446b-aee7-7056b45685d7","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:15:04.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b7638d86-d208-446b-aee7-7056b45685d7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:04.677Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b7638d86-d208-446b-aee7-7056b45685d7","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:15:04.677Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b7638d86-d208-446b-aee7-7056b45685d7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:15:04.677Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b7638d86-d208-446b-aee7-7056b45685d7","seconds":300}
{"level":"info","ts":"2026-09-18T10:15:13.736Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7f8df8ec-1dff-43c3-8234-bf957fe4d080"}
{"level":"info","ts":"2026-09-18T10:15:13.736Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7f8df8ec-1dff-43c3-8234-bf957fe4d080"}
{"level":"info","ts":"2026-09-18T10:15:13.736Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7f8df8ec-1dff-43c3-8234-bf957fe4d080"}
{"level":"info","ts":"2026-09-18T10:15:13.736Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7f8df8ec-1dff-43c3-8234-bf957fe4d080","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:15:13.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7f8df8ec-1dff-43c3-8234-bf957fe4d080","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:15:13.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7f8df8ec-1dff-43c3-8234-bf957fe4d080","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:13.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7f8df8ec-1dff-43c3-8234-bf957fe4d080","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:15:13.737Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7f8df8ec-1dff-43c3-8234-bf957fe4d080","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:13.737Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7f8df8ec-1dff-43c3-8234-bf957fe4d080","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:13.737Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7f8df8ec-1dff-43c3-8234-bf957fe4d080","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:13.737Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7f8df8ec-1dff-43c3-8234-bf957fe4d080","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:15:13.737Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7f8df8ec-1dff-43c3-8234-bf957fe4d080","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:15:13.737Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7f8df8ec-1dff-43c3-8234-bf957fe4d080","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:13.737Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7f8df8ec-1dff-43c3-8234-bf957fe4d080","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:15:13.737Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7f8df8ec-1dff-43c3-8234-bf957fe4d080","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:15:13.737Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7f8df8ec-1dff-43c3-8234-bf957fe4d080","seconds":300}
{"level":"info","ts":"2026-09-18T10:15:14.730Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"40d15c25-67ff-43d9-b5b0-904ef4f35b96"}
{"level":"info","ts":"2026-09-18T10:15:14.730Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"40d15c25-67ff-43d9-b5b0-904ef4f35b96"}
{"level":"info","ts":"2026-09-18T10:15:14.730Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"40d15c25-67ff-43d9-b5b0-904ef4f35b96"}
{"level":"info","ts":"2026-09-18T10:15:14.730Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"40d15c25-67ff-43d9-b5b0-904ef4f35b96","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:15:14.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"40d15c25-67ff-43d9-b5b0-904ef4f35b96","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:15:14.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"40d15c25-67ff-43d9-b5b0-904ef4f35b96","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:14.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"40d15c25-67ff-43d9-b5b0-904ef4f35b96","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:15:14.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"40d15c25-67ff-43d9-b5b0-904ef4f35b96","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:14.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"40d15c25-67ff-43d9-b5b0-904ef4f35b96","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:14.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"40d15c25-67ff-43d9-b5b0-904ef4f35b96","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:14.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"40d15c25-67ff-43d9-b5b0-904ef4f35b96","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:15:14.731Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"40d15c25-67ff-43d9-b5b0-904ef4f35b96","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:15:14.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"40d15c25-67ff-43d9-b5b0-904ef4f35b96","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:14.731Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"40d15c25-67ff-43d9-b5b0-904ef4f35b96","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:15:14.766Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"40d15c25-67ff-43d9-b5b0-904ef4f35b96","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:13:47Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:15:14Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-09-18T10:15:14.780Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"559a5c96-f4d8-4052-a18e-5f5017967c92","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:15:14.780Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"14834d9e-08f2-44f4-b457-4506423632b4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:15:14.780Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"559a5c96-f4d8-4052-a18e-5f5017967c92","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:15:14.780Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"559a5c96-f4d8-4052-a18e-5f5017967c92","pod":"ray-cluster-small-group-worker-49ccd","ip":"10.133.0.51"}
{"level":"info","ts":"2026-09-18T10:15:14.780Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"559a5c96-f4d8-4052-a18e-5f5017967c92","pod":"ray-cluster-small-group-worker-zfpb5","ip":"10.133.0.52"}
{"level":"info","ts":"2026-09-18T10:15:14.780Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"559a5c96-f4d8-4052-a18e-5f5017967c92","rayCluster":"ray-cluster","podIPs":["10.132.0.59","10.133.0.51","10.133.0.52"]}
{"level":"info","ts":"2026-09-18T10:15:14.799Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"14834d9e-08f2-44f4-b457-4506423632b4","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:15:14.811Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"14834d9e-08f2-44f4-b457-4506423632b4","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:15:14.811Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"14834d9e-08f2-44f4-b457-4506423632b4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:15:16.780Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4be86e74-f271-40b3-a773-38ada02d4031"}
{"level":"info","ts":"2026-09-18T10:15:16.780Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4be86e74-f271-40b3-a773-38ada02d4031"}
{"level":"info","ts":"2026-09-18T10:15:16.780Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4be86e74-f271-40b3-a773-38ada02d4031"}
{"level":"info","ts":"2026-09-18T10:15:16.780Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4be86e74-f271-40b3-a773-38ada02d4031","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:15:16.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4be86e74-f271-40b3-a773-38ada02d4031","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:15:16.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4be86e74-f271-40b3-a773-38ada02d4031","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:16.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4be86e74-f271-40b3-a773-38ada02d4031","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:15:16.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4be86e74-f271-40b3-a773-38ada02d4031","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:16.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4be86e74-f271-40b3-a773-38ada02d4031","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:16.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4be86e74-f271-40b3-a773-38ada02d4031","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:16.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4be86e74-f271-40b3-a773-38ada02d4031","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:15:16.780Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4be86e74-f271-40b3-a773-38ada02d4031","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:15:16.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4be86e74-f271-40b3-a773-38ada02d4031","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:16.780Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4be86e74-f271-40b3-a773-38ada02d4031","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:15:16.780Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4be86e74-f271-40b3-a773-38ada02d4031","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:15:16.780Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"4be86e74-f271-40b3-a773-38ada02d4031","seconds":300}
{"level":"info","ts":"2026-09-18T10:15:17.755Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"44e5c397-be9e-456d-b44e-507d1326eff0","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:15:17.755Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"44e5c397-be9e-456d-b44e-507d1326eff0","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:15:17.755Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"44e5c397-be9e-456d-b44e-507d1326eff0","pod":"ray-cluster-small-group-worker-49ccd","ip":"10.133.0.51"}
{"level":"info","ts":"2026-09-18T10:15:17.755Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"44e5c397-be9e-456d-b44e-507d1326eff0","pod":"ray-cluster-small-group-worker-zfpb5","ip":"10.133.0.52"}
{"level":"info","ts":"2026-09-18T10:15:17.755Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"44e5c397-be9e-456d-b44e-507d1326eff0","rayCluster":"ray-cluster","podIPs":["10.132.0.59","10.133.0.51","10.133.0.52"]}
{"level":"info","ts":"2026-09-18T10:15:18.748Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ef68dc5d-0bcb-4be6-87ca-63af640f6a41"}
{"level":"info","ts":"2026-09-18T10:15:18.748Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ef68dc5d-0bcb-4be6-87ca-63af640f6a41"}
{"level":"info","ts":"2026-09-18T10:15:18.748Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ef68dc5d-0bcb-4be6-87ca-63af640f6a41"}
{"level":"info","ts":"2026-09-18T10:15:18.748Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ef68dc5d-0bcb-4be6-87ca-63af640f6a41","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:15:18.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ef68dc5d-0bcb-4be6-87ca-63af640f6a41","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:15:18.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ef68dc5d-0bcb-4be6-87ca-63af640f6a41","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:18.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ef68dc5d-0bcb-4be6-87ca-63af640f6a41","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:15:18.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ef68dc5d-0bcb-4be6-87ca-63af640f6a41","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:18.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ef68dc5d-0bcb-4be6-87ca-63af640f6a41","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:18.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ef68dc5d-0bcb-4be6-87ca-63af640f6a41","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:18.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ef68dc5d-0bcb-4be6-87ca-63af640f6a41","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:15:18.748Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ef68dc5d-0bcb-4be6-87ca-63af640f6a41","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:15:18.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ef68dc5d-0bcb-4be6-87ca-63af640f6a41","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:18.749Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ef68dc5d-0bcb-4be6-87ca-63af640f6a41","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:15:18.749Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ef68dc5d-0bcb-4be6-87ca-63af640f6a41","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:15:18.749Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ef68dc5d-0bcb-4be6-87ca-63af640f6a41","seconds":300}
{"level":"info","ts":"2026-09-18T10:15:19.753Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"03de78cd-75a9-4bbe-8d09-817e4155119d"}
{"level":"info","ts":"2026-09-18T10:15:19.753Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"03de78cd-75a9-4bbe-8d09-817e4155119d"}
{"level":"info","ts":"2026-09-18T10:15:19.753Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"03de78cd-75a9-4bbe-8d09-817e4155119d"}
{"level":"info","ts":"2026-09-18T10:15:19.753Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"03de78cd-75a9-4bbe-8d09-817e4155119d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:15:19.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"03de78cd-75a9-4bbe-8d09-817e4155119d","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:15:19.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"03de78cd-75a9-4bbe-8d09-817e4155119d","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:19.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"03de78cd-75a9-4bbe-8d09-817e4155119d","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:15:19.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"03de78cd-75a9-4bbe-8d09-817e4155119d","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:19.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"03de78cd-75a9-4bbe-8d09-817e4155119d","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:19.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"03de78cd-75a9-4bbe-8d09-817e4155119d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:19.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"03de78cd-75a9-4bbe-8d09-817e4155119d","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:15:19.753Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"03de78cd-75a9-4bbe-8d09-817e4155119d","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:15:19.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"03de78cd-75a9-4bbe-8d09-817e4155119d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:19.754Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"03de78cd-75a9-4bbe-8d09-817e4155119d","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:15:19.754Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"03de78cd-75a9-4bbe-8d09-817e4155119d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:15:19.754Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"03de78cd-75a9-4bbe-8d09-817e4155119d","seconds":300}
{"level":"info","ts":"2026-09-18T10:15:25.755Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d6a39f1-255f-4f3e-8759-b3beff4f5e01"}
{"level":"info","ts":"2026-09-18T10:15:25.755Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d6a39f1-255f-4f3e-8759-b3beff4f5e01"}
{"level":"info","ts":"2026-09-18T10:15:25.755Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d6a39f1-255f-4f3e-8759-b3beff4f5e01"}
{"level":"info","ts":"2026-09-18T10:15:25.755Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d6a39f1-255f-4f3e-8759-b3beff4f5e01","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:15:25.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d6a39f1-255f-4f3e-8759-b3beff4f5e01","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:15:25.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d6a39f1-255f-4f3e-8759-b3beff4f5e01","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:25.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d6a39f1-255f-4f3e-8759-b3beff4f5e01","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:15:25.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d6a39f1-255f-4f3e-8759-b3beff4f5e01","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:25.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d6a39f1-255f-4f3e-8759-b3beff4f5e01","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:25.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d6a39f1-255f-4f3e-8759-b3beff4f5e01","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:25.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d6a39f1-255f-4f3e-8759-b3beff4f5e01","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:15:25.756Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d6a39f1-255f-4f3e-8759-b3beff4f5e01","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:15:25.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d6a39f1-255f-4f3e-8759-b3beff4f5e01","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:25.756Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d6a39f1-255f-4f3e-8759-b3beff4f5e01","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:15:25.756Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d6a39f1-255f-4f3e-8759-b3beff4f5e01","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:15:14Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:15:25Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-09-18T10:15:25.771Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"43f0fc88-eb97-4382-ab19-77c1ae40a9c5","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:15:25.771Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"42f2f102-9010-4400-8414-bcb012a1ef7f","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:15:25.771Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"43f0fc88-eb97-4382-ab19-77c1ae40a9c5","pod":"ray-cluster-small-group-worker-49ccd","ip":"10.133.0.51"}
{"level":"info","ts":"2026-09-18T10:15:25.771Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"43f0fc88-eb97-4382-ab19-77c1ae40a9c5","pod":"ray-cluster-small-group-worker-zfpb5","ip":"10.133.0.52"}
{"level":"info","ts":"2026-09-18T10:15:25.771Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"43f0fc88-eb97-4382-ab19-77c1ae40a9c5","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:15:25.771Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"43f0fc88-eb97-4382-ab19-77c1ae40a9c5","rayCluster":"ray-cluster","podIPs":["10.133.0.51","10.133.0.52","10.132.0.59"]}
{"level":"info","ts":"2026-09-18T10:15:25.784Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"42f2f102-9010-4400-8414-bcb012a1ef7f","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:15:25.796Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"42f2f102-9010-4400-8414-bcb012a1ef7f","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:15:25.796Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"42f2f102-9010-4400-8414-bcb012a1ef7f","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:15:27.772Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3208f28b-0680-4928-beca-c6e84373cea9"}
{"level":"info","ts":"2026-09-18T10:15:27.772Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3208f28b-0680-4928-beca-c6e84373cea9"}
{"level":"info","ts":"2026-09-18T10:15:27.772Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3208f28b-0680-4928-beca-c6e84373cea9"}
{"level":"info","ts":"2026-09-18T10:15:27.772Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3208f28b-0680-4928-beca-c6e84373cea9","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:15:27.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3208f28b-0680-4928-beca-c6e84373cea9","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:15:27.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3208f28b-0680-4928-beca-c6e84373cea9","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:27.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3208f28b-0680-4928-beca-c6e84373cea9","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:15:27.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3208f28b-0680-4928-beca-c6e84373cea9","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:27.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3208f28b-0680-4928-beca-c6e84373cea9","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:27.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3208f28b-0680-4928-beca-c6e84373cea9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:27.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3208f28b-0680-4928-beca-c6e84373cea9","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:15:27.772Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3208f28b-0680-4928-beca-c6e84373cea9","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:15:27.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3208f28b-0680-4928-beca-c6e84373cea9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:27.772Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3208f28b-0680-4928-beca-c6e84373cea9","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:15:27.772Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3208f28b-0680-4928-beca-c6e84373cea9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:15:27.772Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3208f28b-0680-4928-beca-c6e84373cea9","seconds":300}
{"level":"info","ts":"2026-09-18T10:15:28.805Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b97676cf-2178-471c-9200-afbc0a44c53b"}
{"level":"info","ts":"2026-09-18T10:15:28.805Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b97676cf-2178-471c-9200-afbc0a44c53b"}
{"level":"info","ts":"2026-09-18T10:15:28.805Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b97676cf-2178-471c-9200-afbc0a44c53b"}
{"level":"info","ts":"2026-09-18T10:15:28.805Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b97676cf-2178-471c-9200-afbc0a44c53b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:15:28.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b97676cf-2178-471c-9200-afbc0a44c53b","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:15:28.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b97676cf-2178-471c-9200-afbc0a44c53b","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:28.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b97676cf-2178-471c-9200-afbc0a44c53b","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:15:28.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b97676cf-2178-471c-9200-afbc0a44c53b","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:28.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b97676cf-2178-471c-9200-afbc0a44c53b","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:28.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b97676cf-2178-471c-9200-afbc0a44c53b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:28.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b97676cf-2178-471c-9200-afbc0a44c53b","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:15:28.806Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b97676cf-2178-471c-9200-afbc0a44c53b","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:15:28.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b97676cf-2178-471c-9200-afbc0a44c53b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:28.806Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b97676cf-2178-471c-9200-afbc0a44c53b","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:15:28.806Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b97676cf-2178-471c-9200-afbc0a44c53b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:15:28.806Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b97676cf-2178-471c-9200-afbc0a44c53b","seconds":300}
{"level":"info","ts":"2026-09-18T10:15:29.798Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"364469d8-c926-4804-86e6-6a854d84b306"}
{"level":"info","ts":"2026-09-18T10:15:29.798Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"364469d8-c926-4804-86e6-6a854d84b306"}
{"level":"info","ts":"2026-09-18T10:15:29.798Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"364469d8-c926-4804-86e6-6a854d84b306"}
{"level":"info","ts":"2026-09-18T10:15:29.798Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"364469d8-c926-4804-86e6-6a854d84b306","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:15:29.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"364469d8-c926-4804-86e6-6a854d84b306","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:15:29.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"364469d8-c926-4804-86e6-6a854d84b306","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:29.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"364469d8-c926-4804-86e6-6a854d84b306","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:15:29.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"364469d8-c926-4804-86e6-6a854d84b306","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:29.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"364469d8-c926-4804-86e6-6a854d84b306","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:29.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"364469d8-c926-4804-86e6-6a854d84b306","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:29.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"364469d8-c926-4804-86e6-6a854d84b306","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:15:29.798Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"364469d8-c926-4804-86e6-6a854d84b306","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:15:29.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"364469d8-c926-4804-86e6-6a854d84b306","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:29.799Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"364469d8-c926-4804-86e6-6a854d84b306","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:13:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:15:29.799Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"364469d8-c926-4804-86e6-6a854d84b306","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:15:25Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:15:29Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-09-18T10:15:29.814Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b56cdd76-b6c2-4a85-9d1a-ff037afc3f51","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:15:29.814Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"5706c595-78d2-41f6-a384-07597fe3b2cd","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:15:29.814Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b56cdd76-b6c2-4a85-9d1a-ff037afc3f51","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:15:29.814Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b56cdd76-b6c2-4a85-9d1a-ff037afc3f51","pod":"ray-cluster-small-group-worker-49ccd","ip":"10.133.0.51"}
{"level":"info","ts":"2026-09-18T10:15:29.814Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b56cdd76-b6c2-4a85-9d1a-ff037afc3f51","pod":"ray-cluster-small-group-worker-zfpb5","ip":"10.133.0.52"}
{"level":"info","ts":"2026-09-18T10:15:29.814Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b56cdd76-b6c2-4a85-9d1a-ff037afc3f51","rayCluster":"ray-cluster","podIPs":["10.132.0.59","10.133.0.51","10.133.0.52"]}
{"level":"info","ts":"2026-09-18T10:15:29.826Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"5706c595-78d2-41f6-a384-07597fe3b2cd","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:15:29.837Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"5706c595-78d2-41f6-a384-07597fe3b2cd","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:15:29.837Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"5706c595-78d2-41f6-a384-07597fe3b2cd","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:15:31.815Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"700ab679-dd28-4d9e-8257-c5b1cfe0e28d"}
{"level":"info","ts":"2026-09-18T10:15:31.815Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"700ab679-dd28-4d9e-8257-c5b1cfe0e28d"}
{"level":"info","ts":"2026-09-18T10:15:31.815Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"700ab679-dd28-4d9e-8257-c5b1cfe0e28d"}
{"level":"info","ts":"2026-09-18T10:15:31.815Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"700ab679-dd28-4d9e-8257-c5b1cfe0e28d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:15:31.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"700ab679-dd28-4d9e-8257-c5b1cfe0e28d","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:15:31.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"700ab679-dd28-4d9e-8257-c5b1cfe0e28d","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:31.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"700ab679-dd28-4d9e-8257-c5b1cfe0e28d","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:15:31.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"700ab679-dd28-4d9e-8257-c5b1cfe0e28d","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:31.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"700ab679-dd28-4d9e-8257-c5b1cfe0e28d","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:31.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"700ab679-dd28-4d9e-8257-c5b1cfe0e28d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:31.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"700ab679-dd28-4d9e-8257-c5b1cfe0e28d","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:15:31.815Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"700ab679-dd28-4d9e-8257-c5b1cfe0e28d","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:15:31.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"700ab679-dd28-4d9e-8257-c5b1cfe0e28d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:31.815Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"700ab679-dd28-4d9e-8257-c5b1cfe0e28d","podName":"ray-cluster-small-group-worker-49ccd","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:13:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:15:31.816Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"700ab679-dd28-4d9e-8257-c5b1cfe0e28d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:15:31.816Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"700ab679-dd28-4d9e-8257-c5b1cfe0e28d","seconds":300}
{"level":"info","ts":"2026-09-18T10:15:40.824Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"679ee17a-2bbf-40ca-a901-40c1256973e6"}
{"level":"info","ts":"2026-09-18T10:15:40.824Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"679ee17a-2bbf-40ca-a901-40c1256973e6"}
{"level":"info","ts":"2026-09-18T10:15:40.824Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"679ee17a-2bbf-40ca-a901-40c1256973e6"}
{"level":"info","ts":"2026-09-18T10:15:40.824Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"679ee17a-2bbf-40ca-a901-40c1256973e6","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:15:40.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"679ee17a-2bbf-40ca-a901-40c1256973e6","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:15:40.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"679ee17a-2bbf-40ca-a901-40c1256973e6","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:40.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"679ee17a-2bbf-40ca-a901-40c1256973e6","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:15:40.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"679ee17a-2bbf-40ca-a901-40c1256973e6","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:40.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"679ee17a-2bbf-40ca-a901-40c1256973e6","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:40.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"679ee17a-2bbf-40ca-a901-40c1256973e6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:40.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"679ee17a-2bbf-40ca-a901-40c1256973e6","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:15:40.825Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"679ee17a-2bbf-40ca-a901-40c1256973e6","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:15:40.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"679ee17a-2bbf-40ca-a901-40c1256973e6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:40.825Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"679ee17a-2bbf-40ca-a901-40c1256973e6","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:15:29Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:15:40Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-09-18T10:15:40.838Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d1d0cfb9-c3a4-4328-97bd-cf2a2a445086","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:15:40.838Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b1d776a9-e5fa-432b-9ced-2aad9546ccd1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:15:40.838Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d1d0cfb9-c3a4-4328-97bd-cf2a2a445086","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:15:40.838Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d1d0cfb9-c3a4-4328-97bd-cf2a2a445086","pod":"ray-cluster-small-group-worker-49ccd","ip":"10.133.0.51"}
{"level":"info","ts":"2026-09-18T10:15:40.838Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d1d0cfb9-c3a4-4328-97bd-cf2a2a445086","pod":"ray-cluster-small-group-worker-zfpb5","ip":"10.133.0.52"}
{"level":"info","ts":"2026-09-18T10:15:40.838Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"d1d0cfb9-c3a4-4328-97bd-cf2a2a445086","rayCluster":"ray-cluster","podIPs":["10.132.0.59","10.133.0.51","10.133.0.52"]}
{"level":"info","ts":"2026-09-18T10:15:40.859Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b1d776a9-e5fa-432b-9ced-2aad9546ccd1","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:15:40.870Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b1d776a9-e5fa-432b-9ced-2aad9546ccd1","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:15:40.870Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b1d776a9-e5fa-432b-9ced-2aad9546ccd1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:15:42.838Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9c8c8b05-7e41-494b-aecd-8eee76d75ab7"}
{"level":"info","ts":"2026-09-18T10:15:42.838Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9c8c8b05-7e41-494b-aecd-8eee76d75ab7"}
{"level":"info","ts":"2026-09-18T10:15:42.838Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9c8c8b05-7e41-494b-aecd-8eee76d75ab7"}
{"level":"info","ts":"2026-09-18T10:15:42.838Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9c8c8b05-7e41-494b-aecd-8eee76d75ab7","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:15:42.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9c8c8b05-7e41-494b-aecd-8eee76d75ab7","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:15:42.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9c8c8b05-7e41-494b-aecd-8eee76d75ab7","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:42.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9c8c8b05-7e41-494b-aecd-8eee76d75ab7","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:15:42.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9c8c8b05-7e41-494b-aecd-8eee76d75ab7","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:42.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9c8c8b05-7e41-494b-aecd-8eee76d75ab7","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:15:42.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9c8c8b05-7e41-494b-aecd-8eee76d75ab7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:42.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9c8c8b05-7e41-494b-aecd-8eee76d75ab7","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:15:42.838Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9c8c8b05-7e41-494b-aecd-8eee76d75ab7","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:15:42.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9c8c8b05-7e41-494b-aecd-8eee76d75ab7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:15:42.839Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9c8c8b05-7e41-494b-aecd-8eee76d75ab7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:15:42.839Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9c8c8b05-7e41-494b-aecd-8eee76d75ab7","seconds":300}
{"level":"info","ts":"2026-09-18T10:16:17.755Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"aa4e1773-429b-487c-a084-3fa953926403","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:16:17.756Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"aa4e1773-429b-487c-a084-3fa953926403","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:16:17.756Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"aa4e1773-429b-487c-a084-3fa953926403","pod":"ray-cluster-small-group-worker-49ccd","ip":"10.133.0.51"}
{"level":"info","ts":"2026-09-18T10:16:17.756Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"aa4e1773-429b-487c-a084-3fa953926403","pod":"ray-cluster-small-group-worker-zfpb5","ip":"10.133.0.52"}
{"level":"info","ts":"2026-09-18T10:16:17.756Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"aa4e1773-429b-487c-a084-3fa953926403","rayCluster":"ray-cluster","podIPs":["10.132.0.59","10.133.0.51","10.133.0.52"]}
{"level":"info","ts":"2026-09-18T10:16:39.964Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:16:39.964Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-pmtv8"}
{"level":"info","ts":"2026-09-18T10:16:39.976Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63"}
{"level":"info","ts":"2026-09-18T10:16:39.976Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63"}
{"level":"info","ts":"2026-09-18T10:16:39.976Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63"}
{"level":"info","ts":"2026-09-18T10:16:39.976Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e54f7aa3-0a71-4665-b81b-d15b2a700545","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:16:39.976Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:16:39.976Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:16:39.976Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:16:39.976Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:16:39.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63","worker Pod":"ray-cluster-small-group-worker-49ccd","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-49ccd. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:16:39.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:16:39.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:16:39.977Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63","namespace":"test-ns-pmtv8","name":"ray-cluster-small-group-worker-49ccd"}
{"level":"info","ts":"2026-09-18T10:16:39.977Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"30dbe06f-e6b1-4bf3-b1bc-df995d35d1fd","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:16:39.977Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"6bbe74cf-ad4f-4c2e-8901-734da4f699b0","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pmtv8"}}
{"level":"info","ts":"2026-09-18T10:16:39.977Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"30dbe06f-e6b1-4bf3-b1bc-df995d35d1fd","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:16:39.977Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"30dbe06f-e6b1-4bf3-b1bc-df995d35d1fd","pod":"ray-cluster-small-group-worker-49ccd","ip":"10.133.0.51"}
{"level":"info","ts":"2026-09-18T10:16:39.977Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"30dbe06f-e6b1-4bf3-b1bc-df995d35d1fd","pod":"ray-cluster-small-group-worker-zfpb5","ip":"10.133.0.52"}
{"level":"info","ts":"2026-09-18T10:16:39.977Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"6bbe74cf-ad4f-4c2e-8901-734da4f699b0","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:16:39.977Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"6bbe74cf-ad4f-4c2e-8901-734da4f699b0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:16:39.977Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"30dbe06f-e6b1-4bf3-b1bc-df995d35d1fd","rayCluster":"ray-cluster","podIPs":["10.132.0.59","10.133.0.51","10.133.0.52"]}
{"level":"info","ts":"2026-09-18T10:16:39.996Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:16:39.996Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63","group":"small-group","indices":{"1":true}}
{"level":"info","ts":"2026-09-18T10:16:39.996Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:16:39.996Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:15:40Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:16:39Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-09-18T10:16:40.017Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"6bbe74cf-ad4f-4c2e-8901-734da4f699b0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:16:40.028Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pmtv8","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:16:40.028Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63"}
{"level":"error","ts":"2026-09-18T10:16:40.028Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8d46e7ca-4fac-4e3e-8190-5c924d2beb63","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:16:40.028Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7619c114-22be-405a-a286-40e5b4103c04"}
{"level":"info","ts":"2026-09-18T10:16:40.028Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7619c114-22be-405a-a286-40e5b4103c04"}
{"level":"info","ts":"2026-09-18T10:16:40.028Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7619c114-22be-405a-a286-40e5b4103c04"}
{"level":"info","ts":"2026-09-18T10:16:40.028Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7619c114-22be-405a-a286-40e5b4103c04","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:16:40.028Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7619c114-22be-405a-a286-40e5b4103c04","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:16:40.028Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7619c114-22be-405a-a286-40e5b4103c04","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:16:40.028Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7619c114-22be-405a-a286-40e5b4103c04","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:16:40.028Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7619c114-22be-405a-a286-40e5b4103c04","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:15:40Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:16:40Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-09-18T10:16:40.042Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"62a096b7-671b-4777-a677-d2f97dce7b71","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:16:40.042Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"62a096b7-671b-4777-a677-d2f97dce7b71","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:16:40.042Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"62a096b7-671b-4777-a677-d2f97dce7b71","pod":"ray-cluster-small-group-worker-49ccd","ip":"10.133.0.51"}
{"level":"info","ts":"2026-09-18T10:16:40.042Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"62a096b7-671b-4777-a677-d2f97dce7b71","pod":"ray-cluster-small-group-worker-zfpb5","ip":"10.133.0.52"}
{"level":"info","ts":"2026-09-18T10:16:40.042Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"62a096b7-671b-4777-a677-d2f97dce7b71","rayCluster":"ray-cluster","podIPs":["10.132.0.59","10.133.0.51","10.133.0.52"]}
{"level":"info","ts":"2026-09-18T10:16:40.055Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7619c114-22be-405a-a286-40e5b4103c04","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pmtv8","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:16:40.055Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7619c114-22be-405a-a286-40e5b4103c04"}
{"level":"error","ts":"2026-09-18T10:16:40.055Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7619c114-22be-405a-a286-40e5b4103c04","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:16:40.055Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"269228f3-b3da-40fc-8f94-54a8d5b383cb"}
{"level":"info","ts":"2026-09-18T10:16:40.055Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"269228f3-b3da-40fc-8f94-54a8d5b383cb"}
{"level":"info","ts":"2026-09-18T10:16:40.055Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"269228f3-b3da-40fc-8f94-54a8d5b383cb"}
{"level":"info","ts":"2026-09-18T10:16:40.055Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"269228f3-b3da-40fc-8f94-54a8d5b383cb","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:16:40.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"269228f3-b3da-40fc-8f94-54a8d5b383cb","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:16:40.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"269228f3-b3da-40fc-8f94-54a8d5b383cb","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:16:40.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"269228f3-b3da-40fc-8f94-54a8d5b383cb","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:16:40.056Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"269228f3-b3da-40fc-8f94-54a8d5b383cb","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:15:40Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:16:40Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-09-18T10:16:40.073Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e54f7aa3-0a71-4665-b81b-d15b2a700545","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:16:40.102Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"eb9bd5b5-b730-4961-9a6d-ec4f15e34656","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:16:40.102Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1b6daaa9-9be0-428a-b955-4208d471e772"}
{"level":"info","ts":"2026-09-18T10:16:40.102Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1b6daaa9-9be0-428a-b955-4208d471e772"}
{"level":"info","ts":"2026-09-18T10:16:40.102Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1b6daaa9-9be0-428a-b955-4208d471e772"}
{"level":"info","ts":"2026-09-18T10:16:40.102Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"eb9bd5b5-b730-4961-9a6d-ec4f15e34656","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:16:40.102Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1b6daaa9-9be0-428a-b955-4208d471e772","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:16:40.102Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"eb9bd5b5-b730-4961-9a6d-ec4f15e34656","pod":"ray-cluster-small-group-worker-49ccd","ip":"10.133.0.51"}
{"level":"info","ts":"2026-09-18T10:16:40.102Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"eb9bd5b5-b730-4961-9a6d-ec4f15e34656","pod":"ray-cluster-small-group-worker-zfpb5","ip":"10.133.0.52"}
{"level":"info","ts":"2026-09-18T10:16:40.102Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"eb9bd5b5-b730-4961-9a6d-ec4f15e34656","rayCluster":"ray-cluster","podIPs":["10.132.0.59","10.133.0.51","10.133.0.52"]}
{"level":"info","ts":"2026-09-18T10:16:40.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1b6daaa9-9be0-428a-b955-4208d471e772","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:16:40.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1b6daaa9-9be0-428a-b955-4208d471e772","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:16:40.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1b6daaa9-9be0-428a-b955-4208d471e772","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:16:40.103Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1b6daaa9-9be0-428a-b955-4208d471e772","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:16:40.103Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1b6daaa9-9be0-428a-b955-4208d471e772","seconds":300}
{"level":"info","ts":"2026-09-18T10:16:40.143Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e54f7aa3-0a71-4665-b81b-d15b2a700545","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:16:40.143Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e54f7aa3-0a71-4665-b81b-d15b2a700545","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:16:40.144Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"eff16fc1-2b0f-4367-878c-708cf4807017","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:16:40.170Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"eff16fc1-2b0f-4367-878c-708cf4807017","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:16:40.197Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"eff16fc1-2b0f-4367-878c-708cf4807017","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:16:40.197Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"eff16fc1-2b0f-4367-878c-708cf4807017","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:16:40.473Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"714046de-d712-4d87-8b6c-d945ec2b570e"}
{"level":"info","ts":"2026-09-18T10:16:40.473Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"714046de-d712-4d87-8b6c-d945ec2b570e"}
{"level":"info","ts":"2026-09-18T10:16:40.473Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"714046de-d712-4d87-8b6c-d945ec2b570e"}
{"level":"info","ts":"2026-09-18T10:16:40.473Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"714046de-d712-4d87-8b6c-d945ec2b570e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:16:40.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"714046de-d712-4d87-8b6c-d945ec2b570e","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:16:40.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"714046de-d712-4d87-8b6c-d945ec2b570e","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:16:40.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"714046de-d712-4d87-8b6c-d945ec2b570e","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:16:40.473Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"714046de-d712-4d87-8b6c-d945ec2b570e","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:16:40Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:16:40Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-09-18T10:16:40.487Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e7f09e99-443f-4174-b6c4-7f16ad1fd397","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:16:40.487Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"60fb6edd-d45b-4d0b-8ba9-ffb31b123222","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:16:40.487Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"60fb6edd-d45b-4d0b-8ba9-ffb31b123222","pod":"ray-cluster-small-group-worker-zfpb5","ip":"10.133.0.52"}
{"level":"info","ts":"2026-09-18T10:16:40.487Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"60fb6edd-d45b-4d0b-8ba9-ffb31b123222","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:16:40.487Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"60fb6edd-d45b-4d0b-8ba9-ffb31b123222","pod":"ray-cluster-small-group-worker-49ccd","ip":"10.133.0.51"}
{"level":"info","ts":"2026-09-18T10:16:40.487Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"60fb6edd-d45b-4d0b-8ba9-ffb31b123222","rayCluster":"ray-cluster","podIPs":["10.133.0.52","10.132.0.59","10.133.0.51"]}
{"level":"info","ts":"2026-09-18T10:16:40.499Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e7f09e99-443f-4174-b6c4-7f16ad1fd397","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:16:40.512Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e7f09e99-443f-4174-b6c4-7f16ad1fd397","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:16:40.512Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e7f09e99-443f-4174-b6c4-7f16ad1fd397","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:16:41.095Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"47117582-0327-478e-93c2-d8073b866baa"}
{"level":"info","ts":"2026-09-18T10:16:41.095Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"47117582-0327-478e-93c2-d8073b866baa"}
{"level":"info","ts":"2026-09-18T10:16:41.095Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"47117582-0327-478e-93c2-d8073b866baa"}
{"level":"info","ts":"2026-09-18T10:16:41.095Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"47117582-0327-478e-93c2-d8073b866baa","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:16:41.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"47117582-0327-478e-93c2-d8073b866baa","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:16:41.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"47117582-0327-478e-93c2-d8073b866baa","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:16:41.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"47117582-0327-478e-93c2-d8073b866baa","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:16:41.096Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"47117582-0327-478e-93c2-d8073b866baa","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:16:41.096Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"47117582-0327-478e-93c2-d8073b866baa","seconds":300}
{"level":"info","ts":"2026-09-18T10:16:41.101Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"70620666-d1e0-41a4-a4fe-528eafde7c7e"}
{"level":"info","ts":"2026-09-18T10:16:41.101Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"70620666-d1e0-41a4-a4fe-528eafde7c7e"}
{"level":"info","ts":"2026-09-18T10:16:41.101Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"70620666-d1e0-41a4-a4fe-528eafde7c7e"}
{"level":"info","ts":"2026-09-18T10:16:41.101Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"70620666-d1e0-41a4-a4fe-528eafde7c7e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:16:41.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"70620666-d1e0-41a4-a4fe-528eafde7c7e","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:16:41.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"70620666-d1e0-41a4-a4fe-528eafde7c7e","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:16:41.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"70620666-d1e0-41a4-a4fe-528eafde7c7e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:16:41.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"70620666-d1e0-41a4-a4fe-528eafde7c7e","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:16:41.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"70620666-d1e0-41a4-a4fe-528eafde7c7e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:16:41.101Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"70620666-d1e0-41a4-a4fe-528eafde7c7e","namespace":"test-ns-pmtv8","name":"ray-cluster-small-group-worker-49ccd"}
{"level":"info","ts":"2026-09-18T10:16:41.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"70620666-d1e0-41a4-a4fe-528eafde7c7e","The worker Pod has already been deleted":"ray-cluster-small-group-worker-49ccd"}
{"level":"info","ts":"2026-09-18T10:16:41.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"70620666-d1e0-41a4-a4fe-528eafde7c7e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:16:41.105Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"70620666-d1e0-41a4-a4fe-528eafde7c7e","group":"small-group","indices":{"1":true}}
{"level":"info","ts":"2026-09-18T10:16:41.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"70620666-d1e0-41a4-a4fe-528eafde7c7e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:16:41.106Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"70620666-d1e0-41a4-a4fe-528eafde7c7e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:16:41.106Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"70620666-d1e0-41a4-a4fe-528eafde7c7e","seconds":300}
{"level":"info","ts":"2026-09-18T10:16:42.103Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fb694bd2-3747-48c3-b032-4aeb5be6b56d"}
{"level":"info","ts":"2026-09-18T10:16:42.103Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fb694bd2-3747-48c3-b032-4aeb5be6b56d"}
{"level":"info","ts":"2026-09-18T10:16:42.103Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fb694bd2-3747-48c3-b032-4aeb5be6b56d"}
{"level":"info","ts":"2026-09-18T10:16:42.103Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fb694bd2-3747-48c3-b032-4aeb5be6b56d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:16:42.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fb694bd2-3747-48c3-b032-4aeb5be6b56d","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:16:42.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fb694bd2-3747-48c3-b032-4aeb5be6b56d","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:16:42.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fb694bd2-3747-48c3-b032-4aeb5be6b56d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:16:42.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fb694bd2-3747-48c3-b032-4aeb5be6b56d","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:16:42.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fb694bd2-3747-48c3-b032-4aeb5be6b56d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:16:42.103Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fb694bd2-3747-48c3-b032-4aeb5be6b56d","namespace":"test-ns-pmtv8","name":"ray-cluster-small-group-worker-49ccd"}
{"level":"info","ts":"2026-09-18T10:16:42.107Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fb694bd2-3747-48c3-b032-4aeb5be6b56d","The worker Pod has already been deleted":"ray-cluster-small-group-worker-49ccd"}
{"level":"info","ts":"2026-09-18T10:16:42.107Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fb694bd2-3747-48c3-b032-4aeb5be6b56d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:16:42.107Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fb694bd2-3747-48c3-b032-4aeb5be6b56d","group":"small-group","indices":{"1":true}}
{"level":"info","ts":"2026-09-18T10:16:42.107Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fb694bd2-3747-48c3-b032-4aeb5be6b56d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:16:42.108Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fb694bd2-3747-48c3-b032-4aeb5be6b56d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:16:42.108Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"fb694bd2-3747-48c3-b032-4aeb5be6b56d","seconds":300}
{"level":"info","ts":"2026-09-18T10:17:17.757Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e38658bf-efab-48e6-9542-9388073dea6e","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:17.757Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e38658bf-efab-48e6-9542-9388073dea6e","pod":"ray-cluster-small-group-worker-zfpb5","ip":"10.133.0.52"}
{"level":"info","ts":"2026-09-18T10:17:17.757Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e38658bf-efab-48e6-9542-9388073dea6e","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:17:17.788Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e38658bf-efab-48e6-9542-9388073dea6e","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-pmtv8.svc","ray-cluster-head-svc.test-ns-pmtv8.svc.cluster.local"],"ipAddresses":["10.133.0.52","10.132.0.59","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:17:17.808Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e38658bf-efab-48e6-9542-9388073dea6e","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-pmtv8.svc","ray-cluster-worker-svc.test-ns-pmtv8.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-pmtv8.svc","*.ray-cluster-worker-svc.test-ns-pmtv8.svc.cluster.local","*-worker-*.test-ns-pmtv8.svc","*-worker-*.test-ns-pmtv8.svc.cluster.local"],"ipAddresses":["10.133.0.52","10.132.0.59","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:17:17.808Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"e38658bf-efab-48e6-9542-9388073dea6e","rayCluster":"ray-cluster","podIPs":["10.133.0.52","10.132.0.59"]}
{"level":"info","ts":"2026-09-18T10:17:50.900Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:50.900Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-pmtv8"}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b1812a39-e209-454d-99cf-6053306c6311","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"88ad4f1f-efed-4414-a0bf-e02a0a255b97","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pmtv8"}}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"88ad4f1f-efed-4414-a0bf-e02a0a255b97","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"88ad4f1f-efed-4414-a0bf-e02a0a255b97","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b1812a39-e209-454d-99cf-6053306c6311","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b1812a39-e209-454d-99cf-6053306c6311","pod":"ray-cluster-small-group-worker-zfpb5","ip":"10.133.0.52"}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b1812a39-e209-454d-99cf-6053306c6311","rayCluster":"ray-cluster","podIPs":["10.132.0.59","10.133.0.52"]}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000"}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000"}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000"}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000","worker Pod":"ray-cluster-small-group-worker-zfpb5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-zfpb5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:50.916Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000","namespace":"test-ns-pmtv8","name":"ray-cluster-small-group-worker-zfpb5"}
{"level":"info","ts":"2026-09-18T10:17:50.917Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"fa264394-7e25-4a24-8065-0d76f12506f8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:50.937Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"88ad4f1f-efed-4414-a0bf-e02a0a255b97","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:50.939Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"efe87da2-8116-48d0-98ec-2e21d5e44e9f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:50.939Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"efe87da2-8116-48d0-98ec-2e21d5e44e9f","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:17:50.939Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"efe87da2-8116-48d0-98ec-2e21d5e44e9f","pod":"ray-cluster-small-group-worker-zfpb5","ip":"10.133.0.52"}
{"level":"info","ts":"2026-09-18T10:17:50.939Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"efe87da2-8116-48d0-98ec-2e21d5e44e9f","rayCluster":"ray-cluster","podIPs":["10.132.0.59","10.133.0.52"]}
{"level":"info","ts":"2026-09-18T10:17:50.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:17:50.946Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:17:50.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:50.947Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:16:40Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-09-18T10:17:50.957Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pmtv8","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:17:50.957Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000"}
{"level":"error","ts":"2026-09-18T10:17:50.957Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"cb6522f9-8999-4491-b0cb-df9e1bbdb000","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:50.957Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7bf0c272-44a7-4579-82fe-5ca4c646aa6b"}
{"level":"info","ts":"2026-09-18T10:17:50.957Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7bf0c272-44a7-4579-82fe-5ca4c646aa6b"}
{"level":"info","ts":"2026-09-18T10:17:50.957Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7bf0c272-44a7-4579-82fe-5ca4c646aa6b"}
{"level":"info","ts":"2026-09-18T10:17:50.957Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7bf0c272-44a7-4579-82fe-5ca4c646aa6b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:50.957Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7bf0c272-44a7-4579-82fe-5ca4c646aa6b","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:50.957Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7bf0c272-44a7-4579-82fe-5ca4c646aa6b","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:50.958Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7bf0c272-44a7-4579-82fe-5ca4c646aa6b","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:17:50.958Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7bf0c272-44a7-4579-82fe-5ca4c646aa6b","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:16:40Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-09-18T10:17:50.964Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"fa264394-7e25-4a24-8065-0d76f12506f8","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:17:50.977Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"be640290-468d-407f-9ed5-28af3cfe0964"}
{"level":"info","ts":"2026-09-18T10:17:50.977Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"be640290-468d-407f-9ed5-28af3cfe0964"}
{"level":"info","ts":"2026-09-18T10:17:50.977Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"be640290-468d-407f-9ed5-28af3cfe0964"}
{"level":"info","ts":"2026-09-18T10:17:50.977Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"be640290-468d-407f-9ed5-28af3cfe0964","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:50.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"be640290-468d-407f-9ed5-28af3cfe0964","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:50.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"be640290-468d-407f-9ed5-28af3cfe0964","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:50.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"be640290-468d-407f-9ed5-28af3cfe0964","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:17:50.977Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"be640290-468d-407f-9ed5-28af3cfe0964","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:16:40Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-09-18T10:17:50.980Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"a678dae0-8576-4b6f-8512-02bf448faf9b","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:50.980Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"a678dae0-8576-4b6f-8512-02bf448faf9b","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:17:50.980Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"a678dae0-8576-4b6f-8512-02bf448faf9b","pod":"ray-cluster-small-group-worker-zfpb5","ip":"10.133.0.52"}
{"level":"info","ts":"2026-09-18T10:17:50.980Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"a678dae0-8576-4b6f-8512-02bf448faf9b","rayCluster":"ray-cluster","podIPs":["10.132.0.59","10.133.0.52"]}
{"level":"info","ts":"2026-09-18T10:17:50.984Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"fa264394-7e25-4a24-8065-0d76f12506f8","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:17:50.984Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"fa264394-7e25-4a24-8065-0d76f12506f8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:50.984Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"70856ba9-f3c7-47c7-a736-675175106e39","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.014Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"be640290-468d-407f-9ed5-28af3cfe0964","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pmtv8","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:17:51.014Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"be640290-468d-407f-9ed5-28af3cfe0964"}
{"level":"error","ts":"2026-09-18T10:17:51.014Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"be640290-468d-407f-9ed5-28af3cfe0964","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:51.016Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ec06fbc4-c16f-4295-88e2-3efb1bb474a2"}
{"level":"info","ts":"2026-09-18T10:17:51.016Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ec06fbc4-c16f-4295-88e2-3efb1bb474a2"}
{"level":"info","ts":"2026-09-18T10:17:51.016Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ec06fbc4-c16f-4295-88e2-3efb1bb474a2"}
{"level":"info","ts":"2026-09-18T10:17:51.016Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ec06fbc4-c16f-4295-88e2-3efb1bb474a2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:51.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ec06fbc4-c16f-4295-88e2-3efb1bb474a2","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:51.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ec06fbc4-c16f-4295-88e2-3efb1bb474a2","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:51.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ec06fbc4-c16f-4295-88e2-3efb1bb474a2","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:17:51.016Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ec06fbc4-c16f-4295-88e2-3efb1bb474a2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:17:51.016Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ec06fbc4-c16f-4295-88e2-3efb1bb474a2","seconds":300}
{"level":"info","ts":"2026-09-18T10:17:51.020Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e7cc8f7d-447f-4aba-bd5e-16349c57801f"}
{"level":"info","ts":"2026-09-18T10:17:51.020Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e7cc8f7d-447f-4aba-bd5e-16349c57801f"}
{"level":"info","ts":"2026-09-18T10:17:51.020Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e7cc8f7d-447f-4aba-bd5e-16349c57801f"}
{"level":"info","ts":"2026-09-18T10:17:51.020Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e7cc8f7d-447f-4aba-bd5e-16349c57801f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:51.020Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e7cc8f7d-447f-4aba-bd5e-16349c57801f","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:51.020Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e7cc8f7d-447f-4aba-bd5e-16349c57801f","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:51.020Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e7cc8f7d-447f-4aba-bd5e-16349c57801f","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:17:51.021Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e7cc8f7d-447f-4aba-bd5e-16349c57801f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:17:51.021Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e7cc8f7d-447f-4aba-bd5e-16349c57801f","seconds":300}
{"level":"info","ts":"2026-09-18T10:17:51.023Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"70856ba9-f3c7-47c7-a736-675175106e39","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:17:51.045Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"70856ba9-f3c7-47c7-a736-675175106e39","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:17:51.045Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"70856ba9-f3c7-47c7-a736-675175106e39","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.412Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2770ea2f-286b-4921-a388-627a2c66d0b3"}
{"level":"info","ts":"2026-09-18T10:17:51.412Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2770ea2f-286b-4921-a388-627a2c66d0b3"}
{"level":"info","ts":"2026-09-18T10:17:51.412Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2770ea2f-286b-4921-a388-627a2c66d0b3"}
{"level":"info","ts":"2026-09-18T10:17:51.412Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2770ea2f-286b-4921-a388-627a2c66d0b3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:51.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2770ea2f-286b-4921-a388-627a2c66d0b3","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:51.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2770ea2f-286b-4921-a388-627a2c66d0b3","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:51.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2770ea2f-286b-4921-a388-627a2c66d0b3","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:17:51.413Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"2770ea2f-286b-4921-a388-627a2c66d0b3","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:50Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:51Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-09-18T10:17:51.426Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"c415c367-3f58-43ee-8daa-c8c386fb121b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.426Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"0c7c4a82-7a3b-4997-8ae6-d7713ef9a3b0","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.426Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"0c7c4a82-7a3b-4997-8ae6-d7713ef9a3b0","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:17:51.426Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"0c7c4a82-7a3b-4997-8ae6-d7713ef9a3b0","pod":"ray-cluster-small-group-worker-zfpb5","ip":"10.133.0.52"}
{"level":"info","ts":"2026-09-18T10:17:51.426Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"0c7c4a82-7a3b-4997-8ae6-d7713ef9a3b0","rayCluster":"ray-cluster","podIPs":["10.132.0.59","10.133.0.52"]}
{"level":"info","ts":"2026-09-18T10:17:51.437Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"c415c367-3f58-43ee-8daa-c8c386fb121b","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:17:51.447Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"c415c367-3f58-43ee-8daa-c8c386fb121b","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:17:51.448Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"c415c367-3f58-43ee-8daa-c8c386fb121b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.586Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.586Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-wlgxb"}
{"level":"info","ts":"2026-09-18T10:17:51.596Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"b7b1c5b1-a2f0-4766-9c8a-dcee9cba05bc","namespacedName":{"name":"ray-cluster","namespace":"test-ns-wlgxb"}}
{"level":"info","ts":"2026-09-18T10:17:51.596Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"b7b1c5b1-a2f0-4766-9c8a-dcee9cba05bc","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.596Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"b7b1c5b1-a2f0-4766-9c8a-dcee9cba05bc","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.596Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"b7b1c5b1-a2f0-4766-9c8a-dcee9cba05bc","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.596Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"aaa53847-935a-4728-922c-a94a9bddd29e","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.596Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"aaa53847-935a-4728-922c-a94a9bddd29e","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.596Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"e456b346-7aa1-4a34-a51a-939516b31563","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.602Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.602Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-wlgxb"}
{"level":"info","ts":"2026-09-18T10:17:51.605Z","logger":"controllers.RayCluster","msg":"Created service account for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.609Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"aaa53847-935a-4728-922c-a94a9bddd29e","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.610Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"aaa53847-935a-4728-922c-a94a9bddd29e","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.615Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"e456b346-7aa1-4a34-a51a-939516b31563","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.616Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ce4576eb-dcb7-4d86-a114-09de4addb615","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.621Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"b7b1c5b1-a2f0-4766-9c8a-dcee9cba05bc","name":"ray-cluster-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:17:51.621Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"b7b1c5b1-a2f0-4766-9c8a-dcee9cba05bc","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.626Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"aaa53847-935a-4728-922c-a94a9bddd29e","rayCluster":"ray-cluster","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T10:17:51.626Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"aaa53847-935a-4728-922c-a94a9bddd29e","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.638Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ce4576eb-dcb7-4d86-a114-09de4addb615","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:17:51.639Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"b7b1c5b1-a2f0-4766-9c8a-dcee9cba05bc","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.639Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a533dd80-5909-4327-8252-e2cfa8021785","namespacedName":{"name":"ray-cluster","namespace":"test-ns-wlgxb"}}
{"level":"info","ts":"2026-09-18T10:17:51.639Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a533dd80-5909-4327-8252-e2cfa8021785","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.639Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a533dd80-5909-4327-8252-e2cfa8021785","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.639Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a533dd80-5909-4327-8252-e2cfa8021785","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.643Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"aaa53847-935a-4728-922c-a94a9bddd29e","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.666Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a533dd80-5909-4327-8252-e2cfa8021785","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.666Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"aaa53847-935a-4728-922c-a94a9bddd29e","rayCluster":"ray-cluster","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:17:51.678Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ce4576eb-dcb7-4d86-a114-09de4addb615","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:17:51.678Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ce4576eb-dcb7-4d86-a114-09de4addb615","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.678Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"dbef5104-6294-4ece-b98d-f2a58a1699e1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.679Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"aaa53847-935a-4728-922c-a94a9bddd29e","rayCluster":"ray-cluster","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:17:51.679Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"aaa53847-935a-4728-922c-a94a9bddd29e"}
{"level":"info","ts":"2026-09-18T10:17:51.679Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"7b2ae3dd-8bb7-42e7-99d0-f027dd545a60","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.679Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"7b2ae3dd-8bb7-42e7-99d0-f027dd545a60"}
{"level":"info","ts":"2026-09-18T10:17:51.694Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"dbef5104-6294-4ece-b98d-f2a58a1699e1","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:17:51.708Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"dbef5104-6294-4ece-b98d-f2a58a1699e1","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:17:51.708Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"dbef5104-6294-4ece-b98d-f2a58a1699e1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.780Z","logger":"controllers.RayCluster","msg":"Created role for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.888Z","logger":"controllers.RayCluster","msg":"Created role binding for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.888Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a"}
{"level":"info","ts":"2026-09-18T10:17:51.888Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a"}
{"level":"info","ts":"2026-09-18T10:17:51.888Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a"}
{"level":"info","ts":"2026-09-18T10:17:51.899Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a","name":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:51.899Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:17:51.899Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:51.899Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:17:51.899Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9516863d-2b0c-45c2-b0f9-6b189b680d89","namespacedName":{"name":"ray-cluster","namespace":"test-ns-wlgxb"}}
{"level":"info","ts":"2026-09-18T10:17:51.899Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9516863d-2b0c-45c2-b0f9-6b189b680d89","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.899Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9516863d-2b0c-45c2-b0f9-6b189b680d89","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.899Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:51.899Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:51.899Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:51.899Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:17:51.912Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-wlgxb","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:17:51.912Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a"}
{"level":"error","ts":"2026-09-18T10:17:51.912Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bcfeacdb-e818-473d-92c6-770bcedb095a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:51.912Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64d060c0-016f-4216-a57c-ab95850aa585"}
{"level":"info","ts":"2026-09-18T10:17:51.912Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64d060c0-016f-4216-a57c-ab95850aa585"}
{"level":"info","ts":"2026-09-18T10:17:51.912Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64d060c0-016f-4216-a57c-ab95850aa585"}
{"level":"info","ts":"2026-09-18T10:17:51.912Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64d060c0-016f-4216-a57c-ab95850aa585","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:51.912Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64d060c0-016f-4216-a57c-ab95850aa585","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:17:51.912Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64d060c0-016f-4216-a57c-ab95850aa585","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:51.912Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64d060c0-016f-4216-a57c-ab95850aa585","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:17:51.912Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64d060c0-016f-4216-a57c-ab95850aa585","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:51.912Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64d060c0-016f-4216-a57c-ab95850aa585","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:51.912Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64d060c0-016f-4216-a57c-ab95850aa585","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:51.912Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64d060c0-016f-4216-a57c-ab95850aa585","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:51Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"}]},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:51Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:17:51.924Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64d060c0-016f-4216-a57c-ab95850aa585"}
{"level":"error","ts":"2026-09-18T10:17:51.924Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64d060c0-016f-4216-a57c-ab95850aa585","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:51.924Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"21bfebce-66d2-4dee-b82e-5d82169332f9"}
{"level":"info","ts":"2026-09-18T10:17:51.924Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"21bfebce-66d2-4dee-b82e-5d82169332f9"}
{"level":"info","ts":"2026-09-18T10:17:51.924Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"21bfebce-66d2-4dee-b82e-5d82169332f9"}
{"level":"info","ts":"2026-09-18T10:17:51.924Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"21bfebce-66d2-4dee-b82e-5d82169332f9","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:51.924Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"21bfebce-66d2-4dee-b82e-5d82169332f9","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:17:51.924Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"21bfebce-66d2-4dee-b82e-5d82169332f9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:51.924Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"21bfebce-66d2-4dee-b82e-5d82169332f9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:17:51.924Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"21bfebce-66d2-4dee-b82e-5d82169332f9","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:51.924Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"21bfebce-66d2-4dee-b82e-5d82169332f9","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:51.924Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"21bfebce-66d2-4dee-b82e-5d82169332f9","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:51.924Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"21bfebce-66d2-4dee-b82e-5d82169332f9","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:51Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"}]},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:51Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:17:51.925Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"cfb5a9b1-6e3a-478a-b9ae-e87e7780e7bc","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.925Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"df8d1de2-7dd9-4e7b-83f7-750d293e2343","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.925Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"df8d1de2-7dd9-4e7b-83f7-750d293e2343"}
{"level":"info","ts":"2026-09-18T10:17:51.932Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"21bfebce-66d2-4dee-b82e-5d82169332f9","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-wlgxb","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:17:51.932Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"21bfebce-66d2-4dee-b82e-5d82169332f9"}
{"level":"error","ts":"2026-09-18T10:17:51.932Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"21bfebce-66d2-4dee-b82e-5d82169332f9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:51.935Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"80a7a1eb-835b-41a5-a0b1-eb875e080a27"}
{"level":"info","ts":"2026-09-18T10:17:51.935Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"80a7a1eb-835b-41a5-a0b1-eb875e080a27"}
{"level":"info","ts":"2026-09-18T10:17:51.935Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"80a7a1eb-835b-41a5-a0b1-eb875e080a27"}
{"level":"info","ts":"2026-09-18T10:17:51.935Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"80a7a1eb-835b-41a5-a0b1-eb875e080a27","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:51.935Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"80a7a1eb-835b-41a5-a0b1-eb875e080a27","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:17:51.935Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"80a7a1eb-835b-41a5-a0b1-eb875e080a27","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:51.935Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"80a7a1eb-835b-41a5-a0b1-eb875e080a27","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:17:51.935Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"80a7a1eb-835b-41a5-a0b1-eb875e080a27","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:51.935Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"80a7a1eb-835b-41a5-a0b1-eb875e080a27","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:51.935Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"80a7a1eb-835b-41a5-a0b1-eb875e080a27","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:51.935Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"80a7a1eb-835b-41a5-a0b1-eb875e080a27"}
{"level":"error","ts":"2026-09-18T10:17:51.935Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"80a7a1eb-835b-41a5-a0b1-eb875e080a27","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:51.941Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"cfb5a9b1-6e3a-478a-b9ae-e87e7780e7bc","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:17:51.956Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"cfb5a9b1-6e3a-478a-b9ae-e87e7780e7bc","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:17:51.956Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"cfb5a9b1-6e3a-478a-b9ae-e87e7780e7bc","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:51.976Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6e352548-1929-40cd-8e0c-bad320cbb8a0"}
{"level":"info","ts":"2026-09-18T10:17:51.976Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6e352548-1929-40cd-8e0c-bad320cbb8a0"}
{"level":"info","ts":"2026-09-18T10:17:51.976Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6e352548-1929-40cd-8e0c-bad320cbb8a0"}
{"level":"info","ts":"2026-09-18T10:17:51.976Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6e352548-1929-40cd-8e0c-bad320cbb8a0","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:51.976Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6e352548-1929-40cd-8e0c-bad320cbb8a0","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:17:51.976Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6e352548-1929-40cd-8e0c-bad320cbb8a0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:51.976Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6e352548-1929-40cd-8e0c-bad320cbb8a0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:17:51.976Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6e352548-1929-40cd-8e0c-bad320cbb8a0","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:51.976Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6e352548-1929-40cd-8e0c-bad320cbb8a0","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:51.976Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6e352548-1929-40cd-8e0c-bad320cbb8a0","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:51.976Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6e352548-1929-40cd-8e0c-bad320cbb8a0"}
{"level":"error","ts":"2026-09-18T10:17:51.976Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6e352548-1929-40cd-8e0c-bad320cbb8a0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:52.019Z","logger":"authentication-controller","msg":"Added auth proxy SA to autoscaler RoleBinding","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9516863d-2b0c-45c2-b0f9-6b189b680d89","roleBinding":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"authServiceAccount":"ray-cluster-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:17:52.025Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9516863d-2b0c-45c2-b0f9-6b189b680d89","name":"kuberay-gateway-access","namespace":"test-ns-wlgxb","operation":"created"}
{"level":"info","ts":"2026-09-18T10:17:52.033Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9516863d-2b0c-45c2-b0f9-6b189b680d89","name":"test-ns-wlgxb-ray-cluster","operation":"created"}
{"level":"info","ts":"2026-09-18T10:17:52.041Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9516863d-2b0c-45c2-b0f9-6b189b680d89","name":"kube-rbac-proxy-config-ray-cluster","operation":"created"}
{"level":"error","ts":"2026-09-18T10:17:52.049Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9516863d-2b0c-45c2-b0f9-6b189b680d89","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:52.049Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9516863d-2b0c-45c2-b0f9-6b189b680d89"}
{"level":"error","ts":"2026-09-18T10:17:52.049Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9516863d-2b0c-45c2-b0f9-6b189b680d89","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:52.054Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"24658c6d-1498-4dab-b1cf-a36d1a79be53","namespacedName":{"name":"ray-cluster","namespace":"test-ns-wlgxb"}}
{"level":"info","ts":"2026-09-18T10:17:52.054Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"24658c6d-1498-4dab-b1cf-a36d1a79be53","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:52.054Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"24658c6d-1498-4dab-b1cf-a36d1a79be53","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:52.056Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f925dcb1-e3d2-4ae2-a6e0-2da9a652d767"}
{"level":"info","ts":"2026-09-18T10:17:52.056Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f925dcb1-e3d2-4ae2-a6e0-2da9a652d767"}
{"level":"info","ts":"2026-09-18T10:17:52.056Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f925dcb1-e3d2-4ae2-a6e0-2da9a652d767"}
{"level":"info","ts":"2026-09-18T10:17:52.056Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f925dcb1-e3d2-4ae2-a6e0-2da9a652d767","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:52.056Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f925dcb1-e3d2-4ae2-a6e0-2da9a652d767","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:17:52.056Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f925dcb1-e3d2-4ae2-a6e0-2da9a652d767","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:52.056Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f925dcb1-e3d2-4ae2-a6e0-2da9a652d767","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:17:52.057Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f925dcb1-e3d2-4ae2-a6e0-2da9a652d767","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:52.057Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f925dcb1-e3d2-4ae2-a6e0-2da9a652d767","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:52.057Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f925dcb1-e3d2-4ae2-a6e0-2da9a652d767","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:52.057Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f925dcb1-e3d2-4ae2-a6e0-2da9a652d767"}
{"level":"error","ts":"2026-09-18T10:17:52.057Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f925dcb1-e3d2-4ae2-a6e0-2da9a652d767","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:52.071Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"24658c6d-1498-4dab-b1cf-a36d1a79be53","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:52.071Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"501c93fb-652c-4071-9ccb-0c3298c92028","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:52.071Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"4262f6a8-55de-4edc-acf6-fec89ea47230","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:52.071Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"501c93fb-652c-4071-9ccb-0c3298c92028"}
{"level":"info","ts":"2026-09-18T10:17:52.085Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"4262f6a8-55de-4edc-acf6-fec89ea47230","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:17:52.099Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"4262f6a8-55de-4edc-acf6-fec89ea47230","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:17:52.099Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"4262f6a8-55de-4edc-acf6-fec89ea47230","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:52.217Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5516849f-4873-46b8-9872-c0477d697000"}
{"level":"info","ts":"2026-09-18T10:17:52.217Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5516849f-4873-46b8-9872-c0477d697000"}
{"level":"info","ts":"2026-09-18T10:17:52.217Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5516849f-4873-46b8-9872-c0477d697000"}
{"level":"info","ts":"2026-09-18T10:17:52.217Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5516849f-4873-46b8-9872-c0477d697000","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:52.217Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5516849f-4873-46b8-9872-c0477d697000","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:17:52.217Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5516849f-4873-46b8-9872-c0477d697000","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:52.217Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5516849f-4873-46b8-9872-c0477d697000","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:17:52.217Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5516849f-4873-46b8-9872-c0477d697000","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:52.217Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5516849f-4873-46b8-9872-c0477d697000","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:52.217Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5516849f-4873-46b8-9872-c0477d697000","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:52.218Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5516849f-4873-46b8-9872-c0477d697000"}
{"level":"error","ts":"2026-09-18T10:17:52.218Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5516849f-4873-46b8-9872-c0477d697000","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:52.390Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7884ea53-d64f-48d9-95d4-dff89682aba9"}
{"level":"info","ts":"2026-09-18T10:17:52.390Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7884ea53-d64f-48d9-95d4-dff89682aba9"}
{"level":"info","ts":"2026-09-18T10:17:52.390Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7884ea53-d64f-48d9-95d4-dff89682aba9"}
{"level":"info","ts":"2026-09-18T10:17:52.390Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7884ea53-d64f-48d9-95d4-dff89682aba9","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:52.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7884ea53-d64f-48d9-95d4-dff89682aba9","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:52.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7884ea53-d64f-48d9-95d4-dff89682aba9","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:52.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7884ea53-d64f-48d9-95d4-dff89682aba9","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:17:52.391Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7884ea53-d64f-48d9-95d4-dff89682aba9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:17:52.391Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"7884ea53-d64f-48d9-95d4-dff89682aba9","seconds":300}
{"level":"info","ts":"2026-09-18T10:17:52.401Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"538132e9-122a-47ad-9820-6168bfb34510"}
{"level":"info","ts":"2026-09-18T10:17:52.401Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"538132e9-122a-47ad-9820-6168bfb34510"}
{"level":"info","ts":"2026-09-18T10:17:52.401Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"538132e9-122a-47ad-9820-6168bfb34510"}
{"level":"info","ts":"2026-09-18T10:17:52.401Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"538132e9-122a-47ad-9820-6168bfb34510","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:52.401Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"538132e9-122a-47ad-9820-6168bfb34510","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:52.401Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"538132e9-122a-47ad-9820-6168bfb34510","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:52.401Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"538132e9-122a-47ad-9820-6168bfb34510","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:17:52.401Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"538132e9-122a-47ad-9820-6168bfb34510","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:52.401Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"538132e9-122a-47ad-9820-6168bfb34510","namespace":"test-ns-pmtv8","name":"ray-cluster-small-group-worker-zfpb5"}
{"level":"info","ts":"2026-09-18T10:17:52.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"538132e9-122a-47ad-9820-6168bfb34510","The worker Pod has already been deleted":"ray-cluster-small-group-worker-zfpb5"}
{"level":"info","ts":"2026-09-18T10:17:52.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"538132e9-122a-47ad-9820-6168bfb34510","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:17:52.405Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"538132e9-122a-47ad-9820-6168bfb34510","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:17:52.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"538132e9-122a-47ad-9820-6168bfb34510","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:52.405Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"538132e9-122a-47ad-9820-6168bfb34510","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:17:52.405Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"538132e9-122a-47ad-9820-6168bfb34510","seconds":300}
{"level":"info","ts":"2026-09-18T10:17:52.538Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4e686d3b-2a07-438d-a7e8-2412dcfddd7a"}
{"level":"info","ts":"2026-09-18T10:17:52.538Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4e686d3b-2a07-438d-a7e8-2412dcfddd7a"}
{"level":"info","ts":"2026-09-18T10:17:52.538Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4e686d3b-2a07-438d-a7e8-2412dcfddd7a"}
{"level":"info","ts":"2026-09-18T10:17:52.538Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4e686d3b-2a07-438d-a7e8-2412dcfddd7a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:52.539Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4e686d3b-2a07-438d-a7e8-2412dcfddd7a","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-09-18T10:17:52.539Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4e686d3b-2a07-438d-a7e8-2412dcfddd7a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:52.539Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4e686d3b-2a07-438d-a7e8-2412dcfddd7a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-09-18T10:17:52.539Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4e686d3b-2a07-438d-a7e8-2412dcfddd7a","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:52.539Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4e686d3b-2a07-438d-a7e8-2412dcfddd7a","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:52.539Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4e686d3b-2a07-438d-a7e8-2412dcfddd7a","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:17:52.539Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4e686d3b-2a07-438d-a7e8-2412dcfddd7a"}
{"level":"error","ts":"2026-09-18T10:17:52.539Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4e686d3b-2a07-438d-a7e8-2412dcfddd7a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:53.179Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09"}
{"level":"info","ts":"2026-09-18T10:17:53.179Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09"}
{"level":"info","ts":"2026-09-18T10:17:53.179Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09"}
{"level":"info","ts":"2026-09-18T10:17:53.179Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:53.179Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09"}
{"level":"info","ts":"2026-09-18T10:17:53.179Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:17:53.180Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09"}
{"level":"info","ts":"2026-09-18T10:17:53.180Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:53.180Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:17:53.180Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:17:53.180Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:17:53.180Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:17:53.180Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:17:53.180Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:17:53.180Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","enabled":true}
{"level":"info","ts":"2026-09-18T10:17:53.203Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","name":"ray-cluster-head-4f7tw"}
{"level":"info","ts":"2026-09-18T10:17:53.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:17:53.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:53.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:17:53.203Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:17:53.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:53.203Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:17:53.203Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:17:53.203Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b437da53-7732-4aa3-96b4-da75291c9b09","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:17:53.219Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a"}
{"level":"info","ts":"2026-09-18T10:17:53.219Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a"}
{"level":"info","ts":"2026-09-18T10:17:53.219Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a"}
{"level":"info","ts":"2026-09-18T10:17:53.219Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:53.219Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:53.219Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:53.219Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:17:53.219Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:53.219Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:17:53.219Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:17:53.219Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8f06c0a8-8950-43a2-83d9-de8259498841","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:53.219Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:53.219Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"acde040a-fab5-48b6-b2bb-eec99d9b3054","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:53.266Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"acde040a-fab5-48b6-b2bb-eec99d9b3054"}
{"level":"info","ts":"2026-09-18T10:17:53.266Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:17:53.266Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:17:53.266Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-wlgxb","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a"}
{"level":"error","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"5824563e-4386-4e09-b687-ce0f38f5a55a","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"465a3d05-c4d5-4fe9-aa7b-bd87bc702996"}
{"level":"info","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"465a3d05-c4d5-4fe9-aa7b-bd87bc702996"}
{"level":"info","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"465a3d05-c4d5-4fe9-aa7b-bd87bc702996"}
{"level":"info","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"465a3d05-c4d5-4fe9-aa7b-bd87bc702996","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"465a3d05-c4d5-4fe9-aa7b-bd87bc702996","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"465a3d05-c4d5-4fe9-aa7b-bd87bc702996","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"465a3d05-c4d5-4fe9-aa7b-bd87bc702996","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"465a3d05-c4d5-4fe9-aa7b-bd87bc702996","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"465a3d05-c4d5-4fe9-aa7b-bd87bc702996","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"465a3d05-c4d5-4fe9-aa7b-bd87bc702996","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"465a3d05-c4d5-4fe9-aa7b-bd87bc702996","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"465a3d05-c4d5-4fe9-aa7b-bd87bc702996","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"465a3d05-c4d5-4fe9-aa7b-bd87bc702996","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:17:53.283Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"465a3d05-c4d5-4fe9-aa7b-bd87bc702996","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]; autoscaler: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:17:53.289Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8f06c0a8-8950-43a2-83d9-de8259498841","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:17:53.302Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"879e599b-3a02-49e6-84c1-1ad1601d6e3a","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:53.302Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"879e599b-3a02-49e6-84c1-1ad1601d6e3a"}
{"level":"info","ts":"2026-09-18T10:17:53.304Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"276786e8-964b-47bf-8697-729486fac946"}
{"level":"info","ts":"2026-09-18T10:17:53.304Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"276786e8-964b-47bf-8697-729486fac946"}
{"level":"info","ts":"2026-09-18T10:17:53.304Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"276786e8-964b-47bf-8697-729486fac946"}
{"level":"info","ts":"2026-09-18T10:17:53.304Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"276786e8-964b-47bf-8697-729486fac946","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:53.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"276786e8-964b-47bf-8697-729486fac946","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:53.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"276786e8-964b-47bf-8697-729486fac946","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:53.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"276786e8-964b-47bf-8697-729486fac946","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:17:53.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"276786e8-964b-47bf-8697-729486fac946","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:53.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"276786e8-964b-47bf-8697-729486fac946","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:17:53.305Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"276786e8-964b-47bf-8697-729486fac946","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:17:53.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"276786e8-964b-47bf-8697-729486fac946","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:53.305Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"276786e8-964b-47bf-8697-729486fac946","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:17:53.305Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"276786e8-964b-47bf-8697-729486fac946","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:17:53.305Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"276786e8-964b-47bf-8697-729486fac946","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:17:53.305Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"276786e8-964b-47bf-8697-729486fac946","seconds":300}
{"level":"info","ts":"2026-09-18T10:17:53.316Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8f06c0a8-8950-43a2-83d9-de8259498841","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:17:53.316Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8f06c0a8-8950-43a2-83d9-de8259498841","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:53.316Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c11e5a27-3915-47c0-a034-fdf7f6f853c1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:53.341Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c11e5a27-3915-47c0-a034-fdf7f6f853c1","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:17:53.360Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c11e5a27-3915-47c0-a034-fdf7f6f853c1","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:17:53.360Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c11e5a27-3915-47c0-a034-fdf7f6f853c1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:53.426Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e75bffb7-83f2-4468-9f37-d3e1e7b59ced"}
{"level":"info","ts":"2026-09-18T10:17:53.426Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e75bffb7-83f2-4468-9f37-d3e1e7b59ced"}
{"level":"info","ts":"2026-09-18T10:17:53.426Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e75bffb7-83f2-4468-9f37-d3e1e7b59ced"}
{"level":"info","ts":"2026-09-18T10:17:53.426Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e75bffb7-83f2-4468-9f37-d3e1e7b59ced","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:53.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e75bffb7-83f2-4468-9f37-d3e1e7b59ced","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:53.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e75bffb7-83f2-4468-9f37-d3e1e7b59ced","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:53.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e75bffb7-83f2-4468-9f37-d3e1e7b59ced","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:17:53.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e75bffb7-83f2-4468-9f37-d3e1e7b59ced","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:53.426Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e75bffb7-83f2-4468-9f37-d3e1e7b59ced","namespace":"test-ns-pmtv8","name":"ray-cluster-small-group-worker-zfpb5"}
{"level":"info","ts":"2026-09-18T10:17:53.430Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e75bffb7-83f2-4468-9f37-d3e1e7b59ced","The worker Pod has already been deleted":"ray-cluster-small-group-worker-zfpb5"}
{"level":"info","ts":"2026-09-18T10:17:53.430Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e75bffb7-83f2-4468-9f37-d3e1e7b59ced","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:17:53.430Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e75bffb7-83f2-4468-9f37-d3e1e7b59ced","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:17:53.430Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e75bffb7-83f2-4468-9f37-d3e1e7b59ced","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:53.430Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e75bffb7-83f2-4468-9f37-d3e1e7b59ced","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:17:53.430Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"e75bffb7-83f2-4468-9f37-d3e1e7b59ced","seconds":300}
{"level":"info","ts":"2026-09-18T10:17:53.805Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f9719fea-536f-4c22-8bb4-f06c58fdafbd"}
{"level":"info","ts":"2026-09-18T10:17:53.805Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f9719fea-536f-4c22-8bb4-f06c58fdafbd"}
{"level":"info","ts":"2026-09-18T10:17:53.805Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f9719fea-536f-4c22-8bb4-f06c58fdafbd"}
{"level":"info","ts":"2026-09-18T10:17:53.805Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f9719fea-536f-4c22-8bb4-f06c58fdafbd","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:53.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f9719fea-536f-4c22-8bb4-f06c58fdafbd","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:53.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f9719fea-536f-4c22-8bb4-f06c58fdafbd","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:53.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f9719fea-536f-4c22-8bb4-f06c58fdafbd","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:17:53.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f9719fea-536f-4c22-8bb4-f06c58fdafbd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:53.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f9719fea-536f-4c22-8bb4-f06c58fdafbd","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:17:53.806Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f9719fea-536f-4c22-8bb4-f06c58fdafbd","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:17:53.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f9719fea-536f-4c22-8bb4-f06c58fdafbd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:53.806Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f9719fea-536f-4c22-8bb4-f06c58fdafbd","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:17:53.806Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f9719fea-536f-4c22-8bb4-f06c58fdafbd","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:17:53.806Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f9719fea-536f-4c22-8bb4-f06c58fdafbd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:17:53.806Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f9719fea-536f-4c22-8bb4-f06c58fdafbd","seconds":300}
{"level":"info","ts":"2026-09-18T10:17:54.414Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4979d5c4-7106-4af2-91a2-8aec4301aeda"}
{"level":"info","ts":"2026-09-18T10:17:54.414Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4979d5c4-7106-4af2-91a2-8aec4301aeda"}
{"level":"info","ts":"2026-09-18T10:17:54.414Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4979d5c4-7106-4af2-91a2-8aec4301aeda"}
{"level":"info","ts":"2026-09-18T10:17:54.414Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4979d5c4-7106-4af2-91a2-8aec4301aeda","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:54.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4979d5c4-7106-4af2-91a2-8aec4301aeda","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:54.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4979d5c4-7106-4af2-91a2-8aec4301aeda","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:54.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4979d5c4-7106-4af2-91a2-8aec4301aeda","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:17:54.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4979d5c4-7106-4af2-91a2-8aec4301aeda","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:54.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4979d5c4-7106-4af2-91a2-8aec4301aeda","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:17:54.414Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4979d5c4-7106-4af2-91a2-8aec4301aeda","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:17:54.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4979d5c4-7106-4af2-91a2-8aec4301aeda","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:54.414Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4979d5c4-7106-4af2-91a2-8aec4301aeda","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:17:54.414Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4979d5c4-7106-4af2-91a2-8aec4301aeda","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:17:54.414Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4979d5c4-7106-4af2-91a2-8aec4301aeda","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]; autoscaler: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]; autoscaler: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:17:54.429Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"389e1191-5285-4acc-8e4d-483f27e773e3","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:54.429Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"6eb09e1a-4a9a-4551-ab05-e113e6f0ae35","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:54.429Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"389e1191-5285-4acc-8e4d-483f27e773e3","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:17:54.441Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"389e1191-5285-4acc-8e4d-483f27e773e3","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-wlgxb.svc","ray-cluster-head-svc.test-ns-wlgxb.svc.cluster.local"],"ipAddresses":["10.133.0.53","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:17:54.444Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"6eb09e1a-4a9a-4551-ab05-e113e6f0ae35","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:17:54.461Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"389e1191-5285-4acc-8e4d-483f27e773e3","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-wlgxb.svc","ray-cluster-worker-svc.test-ns-wlgxb.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-wlgxb.svc","*.ray-cluster-worker-svc.test-ns-wlgxb.svc.cluster.local","*-worker-*.test-ns-wlgxb.svc","*-worker-*.test-ns-wlgxb.svc.cluster.local"],"ipAddresses":["10.133.0.53","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:17:54.461Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"389e1191-5285-4acc-8e4d-483f27e773e3","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-09-18T10:17:54.467Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"6eb09e1a-4a9a-4551-ab05-e113e6f0ae35","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:17:54.467Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"6eb09e1a-4a9a-4551-ab05-e113e6f0ae35","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:17:55.305Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6806062-7cfc-4bd2-a6bf-a97aaf3c2c8a"}
{"level":"info","ts":"2026-09-18T10:17:55.305Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6806062-7cfc-4bd2-a6bf-a97aaf3c2c8a"}
{"level":"info","ts":"2026-09-18T10:17:55.305Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6806062-7cfc-4bd2-a6bf-a97aaf3c2c8a"}
{"level":"info","ts":"2026-09-18T10:17:55.305Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6806062-7cfc-4bd2-a6bf-a97aaf3c2c8a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:55.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6806062-7cfc-4bd2-a6bf-a97aaf3c2c8a","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:55.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6806062-7cfc-4bd2-a6bf-a97aaf3c2c8a","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:55.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6806062-7cfc-4bd2-a6bf-a97aaf3c2c8a","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:17:55.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6806062-7cfc-4bd2-a6bf-a97aaf3c2c8a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:55.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6806062-7cfc-4bd2-a6bf-a97aaf3c2c8a","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:17:55.305Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6806062-7cfc-4bd2-a6bf-a97aaf3c2c8a","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:17:55.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6806062-7cfc-4bd2-a6bf-a97aaf3c2c8a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:55.305Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6806062-7cfc-4bd2-a6bf-a97aaf3c2c8a","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:17:55.305Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6806062-7cfc-4bd2-a6bf-a97aaf3c2c8a","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:17:55.306Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6806062-7cfc-4bd2-a6bf-a97aaf3c2c8a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:17:55.306Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6806062-7cfc-4bd2-a6bf-a97aaf3c2c8a","seconds":300}
{"level":"info","ts":"2026-09-18T10:17:56.588Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ddc3dded-521b-4388-958a-c208399d02ec"}
{"level":"info","ts":"2026-09-18T10:17:56.588Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ddc3dded-521b-4388-958a-c208399d02ec"}
{"level":"info","ts":"2026-09-18T10:17:56.588Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ddc3dded-521b-4388-958a-c208399d02ec"}
{"level":"info","ts":"2026-09-18T10:17:56.588Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ddc3dded-521b-4388-958a-c208399d02ec","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:56.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ddc3dded-521b-4388-958a-c208399d02ec","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:56.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ddc3dded-521b-4388-958a-c208399d02ec","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:56.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ddc3dded-521b-4388-958a-c208399d02ec","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:17:56.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ddc3dded-521b-4388-958a-c208399d02ec","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:56.588Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ddc3dded-521b-4388-958a-c208399d02ec","namespace":"test-ns-pmtv8","name":"ray-cluster-small-group-worker-zfpb5"}
{"level":"info","ts":"2026-09-18T10:17:56.592Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ddc3dded-521b-4388-958a-c208399d02ec","The worker Pod has already been deleted":"ray-cluster-small-group-worker-zfpb5"}
{"level":"info","ts":"2026-09-18T10:17:56.592Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ddc3dded-521b-4388-958a-c208399d02ec","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:17:56.592Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ddc3dded-521b-4388-958a-c208399d02ec","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:17:56.592Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ddc3dded-521b-4388-958a-c208399d02ec","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:56.593Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ddc3dded-521b-4388-958a-c208399d02ec","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:17:56.593Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"ddc3dded-521b-4388-958a-c208399d02ec","seconds":300}
{"level":"info","ts":"2026-09-18T10:17:56.622Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"759d565c-4e98-4d66-a0d3-51a18c9ae54f"}
{"level":"info","ts":"2026-09-18T10:17:56.622Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"759d565c-4e98-4d66-a0d3-51a18c9ae54f"}
{"level":"info","ts":"2026-09-18T10:17:56.622Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"759d565c-4e98-4d66-a0d3-51a18c9ae54f"}
{"level":"info","ts":"2026-09-18T10:17:56.622Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"759d565c-4e98-4d66-a0d3-51a18c9ae54f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:17:56.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"759d565c-4e98-4d66-a0d3-51a18c9ae54f","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:17:56.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"759d565c-4e98-4d66-a0d3-51a18c9ae54f","head Pod":"ray-cluster-head-w667z","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-w667z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:17:56.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"759d565c-4e98-4d66-a0d3-51a18c9ae54f","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:17:56.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"759d565c-4e98-4d66-a0d3-51a18c9ae54f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:56.622Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"759d565c-4e98-4d66-a0d3-51a18c9ae54f","namespace":"test-ns-pmtv8","name":"ray-cluster-small-group-worker-zfpb5"}
{"level":"info","ts":"2026-09-18T10:17:56.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"759d565c-4e98-4d66-a0d3-51a18c9ae54f","The worker Pod has already been deleted":"ray-cluster-small-group-worker-zfpb5"}
{"level":"info","ts":"2026-09-18T10:17:56.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"759d565c-4e98-4d66-a0d3-51a18c9ae54f","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:17:56.626Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"759d565c-4e98-4d66-a0d3-51a18c9ae54f","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:17:56.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"759d565c-4e98-4d66-a0d3-51a18c9ae54f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:17:56.626Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"759d565c-4e98-4d66-a0d3-51a18c9ae54f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:17:56.626Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"759d565c-4e98-4d66-a0d3-51a18c9ae54f","seconds":300}
{"level":"info","ts":"2026-09-18T10:18:17.809Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b6b8c22e-a2fb-4566-b495-6e9daef4114e","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:17.809Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b6b8c22e-a2fb-4566-b495-6e9daef4114e","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:18:17.823Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b6b8c22e-a2fb-4566-b495-6e9daef4114e","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-pmtv8.svc","ray-cluster-head-svc.test-ns-pmtv8.svc.cluster.local"],"ipAddresses":["10.132.0.59","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:18:17.840Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b6b8c22e-a2fb-4566-b495-6e9daef4114e","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-pmtv8.svc","ray-cluster-worker-svc.test-ns-pmtv8.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-pmtv8.svc","*.ray-cluster-worker-svc.test-ns-pmtv8.svc.cluster.local","*-worker-*.test-ns-pmtv8.svc","*-worker-*.test-ns-pmtv8.svc.cluster.local"],"ipAddresses":["10.132.0.59","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:18:17.840Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b6b8c22e-a2fb-4566-b495-6e9daef4114e"}
{"level":"info","ts":"2026-09-18T10:18:21.680Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ca087c27-d465-48c0-9714-1af2044612f7","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:21.680Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ca087c27-d465-48c0-9714-1af2044612f7","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:18:21.680Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ca087c27-d465-48c0-9714-1af2044612f7","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-09-18T10:18:27.263Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b1ea726f-26ae-40d2-9b4e-d944939d435a"}
{"level":"info","ts":"2026-09-18T10:18:27.263Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b1ea726f-26ae-40d2-9b4e-d944939d435a"}
{"level":"info","ts":"2026-09-18T10:18:27.263Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b1ea726f-26ae-40d2-9b4e-d944939d435a"}
{"level":"info","ts":"2026-09-18T10:18:27.263Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b1ea726f-26ae-40d2-9b4e-d944939d435a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:18:27.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b1ea726f-26ae-40d2-9b4e-d944939d435a","Found 1 head Pod":"ray-cluster-head-w667z","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-09-18 10:13:14 +0000 UTC,FinishedAt:2026-09-18 10:17:59 +0000 UTC,ContainerID:cri-o://fe4ebea829a7218b398193ae93b793419f2f163fcc04802e095bc378efef28ba,}"}
{"level":"info","ts":"2026-09-18T10:18:27.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b1ea726f-26ae-40d2-9b4e-d944939d435a","head Pod":"ray-cluster-head-w667z","shouldDelete":true,"reason":"The head Pod ray-cluster-head-w667z status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-09-18T10:18:27.270Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b1ea726f-26ae-40d2-9b4e-d944939d435a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod ray-cluster-head-w667z status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:27.270Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b1ea726f-26ae-40d2-9b4e-d944939d435a","error":"The head Pod ray-cluster-head-w667z status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-09-18T10:18:27.270Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b1ea726f-26ae-40d2-9b4e-d944939d435a","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:51Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:18:27Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:18:27Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-09-18T10:18:27.284Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b1ea726f-26ae-40d2-9b4e-d944939d435a"}
{"level":"error","ts":"2026-09-18T10:18:27.284Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b1ea726f-26ae-40d2-9b4e-d944939d435a","error":"The head Pod ray-cluster-head-w667z status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:27.284Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b19783ea-1e53-4d49-9f05-746e2b2e4f4b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:27.284Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"be7bd480-1182-4f5a-a731-436974c848c7","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:27.284Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"be7bd480-1182-4f5a-a731-436974c848c7","pod":"ray-cluster-head-w667z","ip":"10.132.0.59"}
{"level":"info","ts":"2026-09-18T10:18:27.284Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"be7bd480-1182-4f5a-a731-436974c848c7"}
{"level":"info","ts":"2026-09-18T10:18:27.289Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"53191413-7b1b-4a71-9d45-f4c298ef2d43"}
{"level":"info","ts":"2026-09-18T10:18:27.289Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"53191413-7b1b-4a71-9d45-f4c298ef2d43"}
{"level":"info","ts":"2026-09-18T10:18:27.289Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"53191413-7b1b-4a71-9d45-f4c298ef2d43"}
{"level":"info","ts":"2026-09-18T10:18:27.289Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"53191413-7b1b-4a71-9d45-f4c298ef2d43","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:18:27.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"53191413-7b1b-4a71-9d45-f4c298ef2d43","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-09-18T10:18:27.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"53191413-7b1b-4a71-9d45-f4c298ef2d43","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:18:27.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"53191413-7b1b-4a71-9d45-f4c298ef2d43","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:18:27.289Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"53191413-7b1b-4a71-9d45-f4c298ef2d43","namespace":"test-ns-pmtv8","name":"ray-cluster-small-group-worker-zfpb5"}
{"level":"info","ts":"2026-09-18T10:18:27.290Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b19783ea-1e53-4d49-9f05-746e2b2e4f4b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:27.294Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"53191413-7b1b-4a71-9d45-f4c298ef2d43","The worker Pod has already been deleted":"ray-cluster-small-group-worker-zfpb5"}
{"level":"info","ts":"2026-09-18T10:18:27.294Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"53191413-7b1b-4a71-9d45-f4c298ef2d43","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:18:27.294Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"53191413-7b1b-4a71-9d45-f4c298ef2d43","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:18:27.294Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"53191413-7b1b-4a71-9d45-f4c298ef2d43","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:18:27.294Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"53191413-7b1b-4a71-9d45-f4c298ef2d43","podName":"ray-cluster-head-w667z","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-09-18T10:18:27.294Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"53191413-7b1b-4a71-9d45-f4c298ef2d43","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:18:27.295Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"53191413-7b1b-4a71-9d45-f4c298ef2d43","seconds":300}
{"level":"info","ts":"2026-09-18T10:18:27.614Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6a5f2324-3fcf-4b4c-a295-d32b9bf0a180"}
{"level":"info","ts":"2026-09-18T10:18:27.614Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6a5f2324-3fcf-4b4c-a295-d32b9bf0a180"}
{"level":"info","ts":"2026-09-18T10:18:27.614Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6a5f2324-3fcf-4b4c-a295-d32b9bf0a180"}
{"level":"info","ts":"2026-09-18T10:18:27.615Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6a5f2324-3fcf-4b4c-a295-d32b9bf0a180","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:18:27.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6a5f2324-3fcf-4b4c-a295-d32b9bf0a180","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-09-18T10:18:27.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6a5f2324-3fcf-4b4c-a295-d32b9bf0a180","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:18:27.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6a5f2324-3fcf-4b4c-a295-d32b9bf0a180","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:18:27.615Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6a5f2324-3fcf-4b4c-a295-d32b9bf0a180","namespace":"test-ns-pmtv8","name":"ray-cluster-small-group-worker-zfpb5"}
{"level":"info","ts":"2026-09-18T10:18:27.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6a5f2324-3fcf-4b4c-a295-d32b9bf0a180","The worker Pod has already been deleted":"ray-cluster-small-group-worker-zfpb5"}
{"level":"info","ts":"2026-09-18T10:18:27.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6a5f2324-3fcf-4b4c-a295-d32b9bf0a180","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:18:27.618Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6a5f2324-3fcf-4b4c-a295-d32b9bf0a180","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:18:27.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6a5f2324-3fcf-4b4c-a295-d32b9bf0a180","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:18:27.619Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6a5f2324-3fcf-4b4c-a295-d32b9bf0a180","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.619Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6a5f2324-3fcf-4b4c-a295-d32b9bf0a180","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.619Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6a5f2324-3fcf-4b4c-a295-d32b9bf0a180","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.619Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"6a5f2324-3fcf-4b4c-a295-d32b9bf0a180","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:18:27Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:18:27Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:18:27Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:18:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2"}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2"}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2"}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"aad3d247-a061-4e53-9c67-d21fb8ce5255","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2"}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2"}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","enabled":true}
{"level":"info","ts":"2026-09-18T10:18:27.632Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"faf7a229-f2a4-4a49-afc4-880600e7530e","cluster":"ray-cluster"}
{"level":"error","ts":"2026-09-18T10:18:27.636Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:27.636Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:18:27.636Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.636Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.636Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.636Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:18:27Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.59","serviceIP":"10.132.0.59","podName":"ray-cluster-head-w667z","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:18:27Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:18:27Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:18:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T10:18:27Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated"}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-09-18T10:18:27.638Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"faf7a229-f2a4-4a49-afc4-880600e7530e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:27.647Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pmtv8","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:18:27.647Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2"}
{"level":"error","ts":"2026-09-18T10:18:27.647Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"1dbc0ce3-c4dd-4c37-9b83-838db44e3bd2","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:27.647Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"aad3d247-a061-4e53-9c67-d21fb8ce5255","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-pmtv8.svc","ray-cluster-head-svc.test-ns-pmtv8.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:18:27.653Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc"}
{"level":"info","ts":"2026-09-18T10:18:27.653Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc"}
{"level":"info","ts":"2026-09-18T10:18:27.653Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc"}
{"level":"info","ts":"2026-09-18T10:18:27.653Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:18:27.653Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc"}
{"level":"info","ts":"2026-09-18T10:18:27.653Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:27.653Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc"}
{"level":"info","ts":"2026-09-18T10:18:27.653Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:27.653Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:27.653Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:18:27.653Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:18:27.653Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:18:27.653Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:27.653Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:27.653Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","enabled":true}
{"level":"error","ts":"2026-09-18T10:18:27.657Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:27.658Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:18:27.658Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.658Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.658Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.658Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:18:27Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:18:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:18:27Z","stateTransitionTimes":{"ready":"2026-09-18T10:13:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:13:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:18:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:13:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:13:05Z","reason":"RayClusterSuspending","message":""},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T10:18:27Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated"}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-09-18T10:18:27.670Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"aad3d247-a061-4e53-9c67-d21fb8ce5255","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-pmtv8.svc","ray-cluster-worker-svc.test-ns-pmtv8.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-pmtv8.svc","*.ray-cluster-worker-svc.test-ns-pmtv8.svc.cluster.local","*-worker-*.test-ns-pmtv8.svc","*-worker-*.test-ns-pmtv8.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:18:27.670Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"aad3d247-a061-4e53-9c67-d21fb8ce5255"}
{"level":"info","ts":"2026-09-18T10:18:27.766Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"9539626d-5d98-45a5-80b4-1837b237d1d3","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:27.766Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"9539626d-5d98-45a5-80b4-1837b237d1d3"}
{"level":"info","ts":"2026-09-18T10:18:27.767Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc"}
{"level":"error","ts":"2026-09-18T10:18:27.767Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"33b25147-93cd-458b-94c1-d20485d67ccc","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:27.767Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"4ca95d90-2ec2-4b1a-ae83-f30ed2f8f593","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:27.773Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"4ca95d90-2ec2-4b1a-ae83-f30ed2f8f593","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:27.777Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318"}
{"level":"info","ts":"2026-09-18T10:18:27.777Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318"}
{"level":"info","ts":"2026-09-18T10:18:27.777Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318"}
{"level":"info","ts":"2026-09-18T10:18:27.777Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:18:27.777Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318"}
{"level":"info","ts":"2026-09-18T10:18:27.777Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:27.777Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318"}
{"level":"info","ts":"2026-09-18T10:18:27.777Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:27.777Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:27.777Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:18:27.777Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:18:27.777Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:18:27.777Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:27.777Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:27.777Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318","enabled":true}
{"level":"error","ts":"2026-09-18T10:18:27.782Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:27.782Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:18:27.782Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.782Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.782Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.782Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318"}
{"level":"error","ts":"2026-09-18T10:18:27.782Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"8579d9f8-43da-40d7-acca-58c875d0b318","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:27.803Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8"}
{"level":"info","ts":"2026-09-18T10:18:27.803Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8"}
{"level":"info","ts":"2026-09-18T10:18:27.803Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8"}
{"level":"info","ts":"2026-09-18T10:18:27.803Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:18:27.803Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8"}
{"level":"info","ts":"2026-09-18T10:18:27.803Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:27.803Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8"}
{"level":"info","ts":"2026-09-18T10:18:27.803Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:27.803Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:27.803Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:18:27.803Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:18:27.803Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:18:27.803Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:27.803Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:27.803Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8","enabled":true}
{"level":"error","ts":"2026-09-18T10:18:27.807Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:27.807Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:18:27.808Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.808Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.808Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.808Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8"}
{"level":"error","ts":"2026-09-18T10:18:27.808Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"bcd07074-e92f-4a60-8588-0787d0e0b8a8","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:27.849Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c"}
{"level":"info","ts":"2026-09-18T10:18:27.849Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c"}
{"level":"info","ts":"2026-09-18T10:18:27.849Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c"}
{"level":"info","ts":"2026-09-18T10:18:27.849Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:18:27.849Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c"}
{"level":"info","ts":"2026-09-18T10:18:27.849Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:27.849Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c"}
{"level":"info","ts":"2026-09-18T10:18:27.849Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:27.849Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:27.849Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:18:27.849Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:18:27.849Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:18:27.849Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:27.849Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:27.849Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c","enabled":true}
{"level":"error","ts":"2026-09-18T10:18:27.854Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:27.854Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:18:27.854Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.854Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.854Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.854Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c"}
{"level":"error","ts":"2026-09-18T10:18:27.854Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"a6f11ed6-6218-4130-a496-1c19ec20267c","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:27.934Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11"}
{"level":"info","ts":"2026-09-18T10:18:27.934Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11"}
{"level":"info","ts":"2026-09-18T10:18:27.934Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11"}
{"level":"info","ts":"2026-09-18T10:18:27.934Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:18:27.935Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11"}
{"level":"info","ts":"2026-09-18T10:18:27.935Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:27.935Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11"}
{"level":"info","ts":"2026-09-18T10:18:27.935Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:27.935Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:27.935Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:18:27.935Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:18:27.935Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:18:27.935Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:27.935Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:27.935Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11","enabled":true}
{"level":"error","ts":"2026-09-18T10:18:27.939Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:27.939Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:18:27.939Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.939Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.939Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:27.939Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11"}
{"level":"error","ts":"2026-09-18T10:18:27.939Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"9873830c-e326-4b3f-952a-6ed644417b11","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:28.100Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1"}
{"level":"info","ts":"2026-09-18T10:18:28.100Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1"}
{"level":"info","ts":"2026-09-18T10:18:28.100Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1"}
{"level":"info","ts":"2026-09-18T10:18:28.100Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:18:28.100Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1"}
{"level":"info","ts":"2026-09-18T10:18:28.100Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:28.100Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1"}
{"level":"info","ts":"2026-09-18T10:18:28.100Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:28.100Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:28.100Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:18:28.100Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:18:28.100Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:18:28.100Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:28.100Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:28.100Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1","enabled":true}
{"level":"error","ts":"2026-09-18T10:18:28.104Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:28.104Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:18:28.104Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:28.104Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:28.104Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:28.104Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1"}
{"level":"error","ts":"2026-09-18T10:18:28.104Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"22f11571-0733-49ee-83f2-f4f88e2a35b1","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:28.426Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96"}
{"level":"info","ts":"2026-09-18T10:18:28.426Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96"}
{"level":"info","ts":"2026-09-18T10:18:28.426Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96"}
{"level":"info","ts":"2026-09-18T10:18:28.426Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:18:28.426Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96"}
{"level":"info","ts":"2026-09-18T10:18:28.426Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:28.426Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96"}
{"level":"info","ts":"2026-09-18T10:18:28.426Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:28.426Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:28.426Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:18:28.426Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:18:28.426Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:18:28.426Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:28.426Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:28.426Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96","enabled":true}
{"level":"error","ts":"2026-09-18T10:18:28.430Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:28.430Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:18:28.430Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:28.430Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:28.430Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:28.430Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96"}
{"level":"error","ts":"2026-09-18T10:18:28.430Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"72aa47e3-146d-42ba-a470-53244eef1d96","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:29.071Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf"}
{"level":"info","ts":"2026-09-18T10:18:29.071Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf"}
{"level":"info","ts":"2026-09-18T10:18:29.071Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf"}
{"level":"info","ts":"2026-09-18T10:18:29.071Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:18:29.072Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf"}
{"level":"info","ts":"2026-09-18T10:18:29.072Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:29.072Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf"}
{"level":"info","ts":"2026-09-18T10:18:29.072Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:29.072Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:29.072Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:18:29.072Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:18:29.072Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:18:29.072Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:29.072Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:29.072Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf","enabled":true}
{"level":"error","ts":"2026-09-18T10:18:29.077Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:29.077Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:18:29.077Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:29.077Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:29.077Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:29.077Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf"}
{"level":"error","ts":"2026-09-18T10:18:29.077Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"b50007e5-f77b-47c9-a798-1c5495bcffbf","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:30.358Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c"}
{"level":"info","ts":"2026-09-18T10:18:30.358Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c"}
{"level":"info","ts":"2026-09-18T10:18:30.358Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c"}
{"level":"info","ts":"2026-09-18T10:18:30.358Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:18:30.358Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c"}
{"level":"info","ts":"2026-09-18T10:18:30.358Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:30.358Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c"}
{"level":"info","ts":"2026-09-18T10:18:30.358Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:30.358Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:30.358Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:18:30.358Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:18:30.358Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:18:30.359Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:30.359Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:30.359Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c","enabled":true}
{"level":"error","ts":"2026-09-18T10:18:30.363Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:30.363Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:18:30.363Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:30.363Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:30.363Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:30.363Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c"}
{"level":"error","ts":"2026-09-18T10:18:30.363Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"3c859029-6ccf-4b5b-ab43-ced2281afe7c","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:32.924Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f"}
{"level":"info","ts":"2026-09-18T10:18:32.924Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f"}
{"level":"info","ts":"2026-09-18T10:18:32.924Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f"}
{"level":"info","ts":"2026-09-18T10:18:32.924Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:18:32.925Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f"}
{"level":"info","ts":"2026-09-18T10:18:32.925Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:32.925Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f"}
{"level":"info","ts":"2026-09-18T10:18:32.925Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:32.925Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:32.925Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:18:32.925Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:18:32.925Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:18:32.925Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:32.925Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:18:32.925Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f","enabled":true}
{"level":"error","ts":"2026-09-18T10:18:32.929Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:32.929Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:18:32.930Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:32.930Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:32.930Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f","filter labels":["test-ns-pmtv8",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:18:32.930Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f"}
{"level":"error","ts":"2026-09-18T10:18:32.930Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"de04f56d-f81d-46ec-be98-3564d70dda4f","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:34.233Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"65b173fd-678d-4862-8a58-e56fd8afad40","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.241Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"65b173fd-678d-4862-8a58-e56fd8afad40","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.242Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"dde1051d-cec5-4fab-8c46-3688168761f7","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.248Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"dde1051d-cec5-4fab-8c46-3688168761f7","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.285Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"reconcileID":"812b0c28-bfb3-49e4-857c-4504439ea163"}
{"level":"info","ts":"2026-09-18T10:18:34.285Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"67d0cf83-a2b3-4f7a-a6ca-4e002b4882cd","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.285Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"67d0cf83-a2b3-4f7a-a6ca-4e002b4882cd","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.285Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"95e80971-6d7d-4308-b5e2-c5fc02087df2"}
{"level":"info","ts":"2026-09-18T10:18:34.285Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"cd2b9feb-e705-4cf4-a729-76df75f331c6","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pmtv8"}}
{"level":"info","ts":"2026-09-18T10:18:34.285Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"cd2b9feb-e705-4cf4-a729-76df75f331c6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.285Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"cd2b9feb-e705-4cf4-a729-76df75f331c6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.286Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"cd2b9feb-e705-4cf4-a729-76df75f331c6","namespace":"test-ns-pmtv8","otherAuthClusters":0,"action":"will delete"}
{"level":"error","ts":"2026-09-18T10:18:34.291Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"67d0cf83-a2b3-4f7a-a6ca-4e002b4882cd","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:18:34.291Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"67d0cf83-a2b3-4f7a-a6ca-4e002b4882cd","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:34.291Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"67d0cf83-a2b3-4f7a-a6ca-4e002b4882cd"}
{"level":"error","ts":"2026-09-18T10:18:34.291Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"67d0cf83-a2b3-4f7a-a6ca-4e002b4882cd","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:34.296Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"7ce38627-5243-4a68-b719-376473d7b060","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.296Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"7ce38627-5243-4a68-b719-376473d7b060","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.299Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"cd2b9feb-e705-4cf4-a729-76df75f331c6","name":"kuberay-gateway-access","namespace":"test-ns-pmtv8"}
{"level":"error","ts":"2026-09-18T10:18:34.301Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"7ce38627-5243-4a68-b719-376473d7b060","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:18:34.301Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"7ce38627-5243-4a68-b719-376473d7b060","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:34.301Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"7ce38627-5243-4a68-b719-376473d7b060"}
{"level":"error","ts":"2026-09-18T10:18:34.301Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"7ce38627-5243-4a68-b719-376473d7b060","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:34.312Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"76d07d49-998e-4c9f-930b-8e1beacfc56d","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.312Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"76d07d49-998e-4c9f-930b-8e1beacfc56d","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.314Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"cd2b9feb-e705-4cf4-a729-76df75f331c6","name":"test-ns-pmtv8-ray-cluster","namespace":"opendatahub"}
{"level":"error","ts":"2026-09-18T10:18:34.321Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"76d07d49-998e-4c9f-930b-8e1beacfc56d","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:18:34.321Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"76d07d49-998e-4c9f-930b-8e1beacfc56d","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:34.321Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"76d07d49-998e-4c9f-930b-8e1beacfc56d"}
{"level":"error","ts":"2026-09-18T10:18:34.321Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"76d07d49-998e-4c9f-930b-8e1beacfc56d","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:34.327Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"cd2b9feb-e705-4cf4-a729-76df75f331c6","configMap":"kube-rbac-proxy-config-ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.334Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"cd2b9feb-e705-4cf4-a729-76df75f331c6","serviceAccount":"ray-cluster-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:18:34.342Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"023c015d-b8e3-4a80-bfb1-ca0e0222c33d","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.342Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"023c015d-b8e3-4a80-bfb1-ca0e0222c33d","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.345Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.345Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-pmtv8"}
{"level":"error","ts":"2026-09-18T10:18:34.350Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"023c015d-b8e3-4a80-bfb1-ca0e0222c33d","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:18:34.350Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"023c015d-b8e3-4a80-bfb1-ca0e0222c33d","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:34.350Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"023c015d-b8e3-4a80-bfb1-ca0e0222c33d"}
{"level":"error","ts":"2026-09-18T10:18:34.350Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"023c015d-b8e3-4a80-bfb1-ca0e0222c33d","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-pmtv8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:18:34.379Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"9311f2d9-6fb9-49b0-b042-b16a2458cc47","namespace":"test-ns-pmtv8","clusterName":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.379Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"18300b45-cb87-4305-8696-6db0861d03fb"}
{"level":"info","ts":"2026-09-18T10:18:34.380Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"cd2b9feb-e705-4cf4-a729-76df75f331c6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.380Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"5f2f67f4-62f5-4706-b0a4-3d6349f83f0c","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pmtv8"}}
{"level":"info","ts":"2026-09-18T10:18:34.380Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"5f2f67f4-62f5-4706-b0a4-3d6349f83f0c"}
{"level":"info","ts":"2026-09-18T10:18:34.380Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"5f2f67f4-62f5-4706-b0a4-3d6349f83f0c","namespace":"test-ns-pmtv8","authClusters":0}
{"level":"info","ts":"2026-09-18T10:18:34.380Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"5f2f67f4-62f5-4706-b0a4-3d6349f83f0c","namespace":"test-ns-pmtv8"}
{"level":"info","ts":"2026-09-18T10:18:34.401Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1fbcddd1-ef84-46be-81e8-70484bf1b81c","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pmtv8"}}
{"level":"info","ts":"2026-09-18T10:18:34.401Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1fbcddd1-ef84-46be-81e8-70484bf1b81c"}
{"level":"info","ts":"2026-09-18T10:18:34.401Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1fbcddd1-ef84-46be-81e8-70484bf1b81c","namespace":"test-ns-pmtv8","authClusters":0}
{"level":"info","ts":"2026-09-18T10:18:34.401Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"1fbcddd1-ef84-46be-81e8-70484bf1b81c","namespace":"test-ns-pmtv8"}
{"level":"info","ts":"2026-09-18T10:18:34.401Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b1f1c5bd-0120-42e6-ab7c-fb54f64fddce","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pmtv8"}}
{"level":"info","ts":"2026-09-18T10:18:34.402Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b1f1c5bd-0120-42e6-ab7c-fb54f64fddce"}
{"level":"info","ts":"2026-09-18T10:18:34.402Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b1f1c5bd-0120-42e6-ab7c-fb54f64fddce","namespace":"test-ns-pmtv8","authClusters":0}
{"level":"info","ts":"2026-09-18T10:18:34.402Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"b1f1c5bd-0120-42e6-ab7c-fb54f64fddce","namespace":"test-ns-pmtv8"}
{"level":"info","ts":"2026-09-18T10:18:34.456Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"9311f2d9-6fb9-49b0-b042-b16a2458cc47","namespace":"test-ns-pmtv8","clusterName":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.456Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"0a9498de-5bf8-47b0-9d5d-9d835989ef6d","namespace":"test-ns-pmtv8","clusterName":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:18:34.456Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pmtv8"},"namespace":"test-ns-pmtv8","name":"ray-cluster","reconcileID":"0a9498de-5bf8-47b0-9d5d-9d835989ef6d","namespace":"test-ns-pmtv8","clusterName":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:21.681Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a7ba0708-bdeb-4c8f-8498-48c244511903","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:21.681Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a7ba0708-bdeb-4c8f-8498-48c244511903","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:19:21.681Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a7ba0708-bdeb-4c8f-8498-48c244511903","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-09-18T10:19:24.774Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3d503f2e-a327-4475-8fa1-1b3de41b9627"}
{"level":"info","ts":"2026-09-18T10:19:24.774Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3d503f2e-a327-4475-8fa1-1b3de41b9627"}
{"level":"info","ts":"2026-09-18T10:19:24.774Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3d503f2e-a327-4475-8fa1-1b3de41b9627"}
{"level":"info","ts":"2026-09-18T10:19:24.774Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3d503f2e-a327-4475-8fa1-1b3de41b9627","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:24.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3d503f2e-a327-4475-8fa1-1b3de41b9627","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:24.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3d503f2e-a327-4475-8fa1-1b3de41b9627","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:24.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3d503f2e-a327-4475-8fa1-1b3de41b9627","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:19:24.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3d503f2e-a327-4475-8fa1-1b3de41b9627","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:24.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3d503f2e-a327-4475-8fa1-1b3de41b9627","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:24.774Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3d503f2e-a327-4475-8fa1-1b3de41b9627","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:19:24.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3d503f2e-a327-4475-8fa1-1b3de41b9627","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:24.774Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3d503f2e-a327-4475-8fa1-1b3de41b9627","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:19:24.774Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3d503f2e-a327-4475-8fa1-1b3de41b9627","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:19:24.774Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3d503f2e-a327-4475-8fa1-1b3de41b9627","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:19:24.774Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3d503f2e-a327-4475-8fa1-1b3de41b9627","seconds":300}
{"level":"info","ts":"2026-09-18T10:19:25.782Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"596d78ff-2c8d-4699-aa75-c15e7cd9d217"}
{"level":"info","ts":"2026-09-18T10:19:25.782Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"596d78ff-2c8d-4699-aa75-c15e7cd9d217"}
{"level":"info","ts":"2026-09-18T10:19:25.782Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"596d78ff-2c8d-4699-aa75-c15e7cd9d217"}
{"level":"info","ts":"2026-09-18T10:19:25.782Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"596d78ff-2c8d-4699-aa75-c15e7cd9d217","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:25.782Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"596d78ff-2c8d-4699-aa75-c15e7cd9d217","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:25.782Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"596d78ff-2c8d-4699-aa75-c15e7cd9d217","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:25.782Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"596d78ff-2c8d-4699-aa75-c15e7cd9d217","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:19:25.782Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"596d78ff-2c8d-4699-aa75-c15e7cd9d217","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:25.782Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"596d78ff-2c8d-4699-aa75-c15e7cd9d217","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:25.782Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"596d78ff-2c8d-4699-aa75-c15e7cd9d217","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:19:25.782Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"596d78ff-2c8d-4699-aa75-c15e7cd9d217","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:25.782Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"596d78ff-2c8d-4699-aa75-c15e7cd9d217","podName":"ray-cluster-head-4f7tw","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:17:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:19:25.783Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"596d78ff-2c8d-4699-aa75-c15e7cd9d217","podName":"ray-cluster-head-4f7tw","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:17:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:19:25.783Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"596d78ff-2c8d-4699-aa75-c15e7cd9d217","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:17:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]; autoscaler: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:19:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:19:25.797Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"403ef913-4e70-4b5f-958b-b6087bbbff2a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:25.797Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a4f0a020-c54e-487f-88bc-e58866adf643","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:25.797Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a4f0a020-c54e-487f-88bc-e58866adf643","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:19:25.797Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a4f0a020-c54e-487f-88bc-e58866adf643","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-09-18T10:19:25.818Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"403ef913-4e70-4b5f-958b-b6087bbbff2a","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:19:25.830Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"403ef913-4e70-4b5f-958b-b6087bbbff2a","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:19:25.830Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"403ef913-4e70-4b5f-958b-b6087bbbff2a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:27.798Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"60a1681b-b409-4dd7-ae55-d68aaae066dd"}
{"level":"info","ts":"2026-09-18T10:19:27.798Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"60a1681b-b409-4dd7-ae55-d68aaae066dd"}
{"level":"info","ts":"2026-09-18T10:19:27.798Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"60a1681b-b409-4dd7-ae55-d68aaae066dd"}
{"level":"info","ts":"2026-09-18T10:19:27.798Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"60a1681b-b409-4dd7-ae55-d68aaae066dd","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:27.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"60a1681b-b409-4dd7-ae55-d68aaae066dd","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:27.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"60a1681b-b409-4dd7-ae55-d68aaae066dd","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:27.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"60a1681b-b409-4dd7-ae55-d68aaae066dd","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:19:27.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"60a1681b-b409-4dd7-ae55-d68aaae066dd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:27.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"60a1681b-b409-4dd7-ae55-d68aaae066dd","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:27.798Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"60a1681b-b409-4dd7-ae55-d68aaae066dd","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:19:27.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"60a1681b-b409-4dd7-ae55-d68aaae066dd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:27.798Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"60a1681b-b409-4dd7-ae55-d68aaae066dd","podName":"ray-cluster-head-4f7tw","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:17:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:19:27.798Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"60a1681b-b409-4dd7-ae55-d68aaae066dd","podName":"ray-cluster-head-4f7tw","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:17:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:19:27.799Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"60a1681b-b409-4dd7-ae55-d68aaae066dd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:19:27.799Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"60a1681b-b409-4dd7-ae55-d68aaae066dd","seconds":300}
{"level":"info","ts":"2026-09-18T10:19:46.811Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f49a9a4b-6869-45cf-8dc6-c95fc9664e52"}
{"level":"info","ts":"2026-09-18T10:19:46.811Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f49a9a4b-6869-45cf-8dc6-c95fc9664e52"}
{"level":"info","ts":"2026-09-18T10:19:46.811Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f49a9a4b-6869-45cf-8dc6-c95fc9664e52"}
{"level":"info","ts":"2026-09-18T10:19:46.811Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f49a9a4b-6869-45cf-8dc6-c95fc9664e52","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:46.811Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f49a9a4b-6869-45cf-8dc6-c95fc9664e52","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:46.811Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f49a9a4b-6869-45cf-8dc6-c95fc9664e52","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:46.811Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f49a9a4b-6869-45cf-8dc6-c95fc9664e52","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:19:46.811Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f49a9a4b-6869-45cf-8dc6-c95fc9664e52","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:46.811Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f49a9a4b-6869-45cf-8dc6-c95fc9664e52","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:46.811Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f49a9a4b-6869-45cf-8dc6-c95fc9664e52","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:19:46.811Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f49a9a4b-6869-45cf-8dc6-c95fc9664e52","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:46.811Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f49a9a4b-6869-45cf-8dc6-c95fc9664e52","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:19:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:19:46Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:19:46.828Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"af4bcc0e-a121-4861-8202-bf8a76089fc6","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:46.828Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ed97e332-db1a-4d83-9f6e-e81907810962","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:46.828Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"af4bcc0e-a121-4861-8202-bf8a76089fc6","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:19:46.828Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"af4bcc0e-a121-4861-8202-bf8a76089fc6","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-09-18T10:19:46.848Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ed97e332-db1a-4d83-9f6e-e81907810962","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:19:46.861Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ed97e332-db1a-4d83-9f6e-e81907810962","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:19:46.861Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ed97e332-db1a-4d83-9f6e-e81907810962","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:48.829Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"227bc82c-b37f-403c-8925-6bd948612659"}
{"level":"info","ts":"2026-09-18T10:19:48.829Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"227bc82c-b37f-403c-8925-6bd948612659"}
{"level":"info","ts":"2026-09-18T10:19:48.829Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"227bc82c-b37f-403c-8925-6bd948612659"}
{"level":"info","ts":"2026-09-18T10:19:48.829Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"227bc82c-b37f-403c-8925-6bd948612659","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:48.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"227bc82c-b37f-403c-8925-6bd948612659","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:48.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"227bc82c-b37f-403c-8925-6bd948612659","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:48.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"227bc82c-b37f-403c-8925-6bd948612659","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:19:48.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"227bc82c-b37f-403c-8925-6bd948612659","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:48.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"227bc82c-b37f-403c-8925-6bd948612659","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:48.829Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"227bc82c-b37f-403c-8925-6bd948612659","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:19:48.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"227bc82c-b37f-403c-8925-6bd948612659","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:48.829Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"227bc82c-b37f-403c-8925-6bd948612659","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:19:48.829Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"227bc82c-b37f-403c-8925-6bd948612659","seconds":300}
{"level":"info","ts":"2026-09-18T10:19:52.954Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:52.954Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-wlgxb"}
{"level":"info","ts":"2026-09-18T10:19:52.966Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ed99986d-cf59-4b71-88fd-5c10b8ca8280","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:52.966Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664"}
{"level":"info","ts":"2026-09-18T10:19:52.966Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664"}
{"level":"info","ts":"2026-09-18T10:19:52.966Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664"}
{"level":"info","ts":"2026-09-18T10:19:52.966Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8a15c704-f2ef-4878-9d14-358b0ffea273","namespacedName":{"name":"ray-cluster","namespace":"test-ns-wlgxb"}}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8a15c704-f2ef-4878-9d14-358b0ffea273","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8a15c704-f2ef-4878-9d14-358b0ffea273","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664"}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","nodeType":"worker","rayStartParams":{"address":"ray-cluster-head-svc.test-ns-wlgxb.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"1G"},"requests":{"cpu":"1","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","rayStartCmd":"ray start  --address=ray-cluster-head-svc.test-ns-wlgxb.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=ray-cluster-head-svc.test-ns-wlgxb.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","enabled":true}
{"level":"info","ts":"2026-09-18T10:19:52.966Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"15fdf2f0-64cb-4231-b842-1106bdaaefbd","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"15fdf2f0-64cb-4231-b842-1106bdaaefbd","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:19:52.967Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"15fdf2f0-64cb-4231-b842-1106bdaaefbd","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-09-18T10:19:52.979Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ed99986d-cf59-4b71-88fd-5c10b8ca8280","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:19:52.981Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8996345d-348e-4f38-ac39-9cc867179ea1","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:52.981Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8996345d-348e-4f38-ac39-9cc867179ea1","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:19:52.981Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8996345d-348e-4f38-ac39-9cc867179ea1","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-09-18T10:19:52.982Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8a15c704-f2ef-4878-9d14-358b0ffea273","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:52.988Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","name":"ray-cluster-small-group-worker-9c9dl"}
{"level":"info","ts":"2026-09-18T10:19:52.988Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:19:46Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:19:52Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-09-18T10:19:52.992Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ed99986d-cf59-4b71-88fd-5c10b8ca8280","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:19:52.992Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ed99986d-cf59-4b71-88fd-5c10b8ca8280","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:52.992Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"bf4fa393-c015-437c-b58d-b903c8401b73","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-wlgxb","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664"}
{"level":"error","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ae2e0be4-148b-4a6e-97c2-7bce6418e664","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a8331e84-5369-40a0-a1cd-5474c7d9198e"}
{"level":"info","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a8331e84-5369-40a0-a1cd-5474c7d9198e"}
{"level":"info","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a8331e84-5369-40a0-a1cd-5474c7d9198e"}
{"level":"info","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a8331e84-5369-40a0-a1cd-5474c7d9198e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a8331e84-5369-40a0-a1cd-5474c7d9198e","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a8331e84-5369-40a0-a1cd-5474c7d9198e","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a8331e84-5369-40a0-a1cd-5474c7d9198e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a8331e84-5369-40a0-a1cd-5474c7d9198e","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a8331e84-5369-40a0-a1cd-5474c7d9198e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a8331e84-5369-40a0-a1cd-5474c7d9198e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a8331e84-5369-40a0-a1cd-5474c7d9198e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a8331e84-5369-40a0-a1cd-5474c7d9198e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a8331e84-5369-40a0-a1cd-5474c7d9198e","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:19:52.999Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a8331e84-5369-40a0-a1cd-5474c7d9198e","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:19:46Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:19:52Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-09-18T10:19:53.012Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"bf4fa393-c015-437c-b58d-b903c8401b73","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:19:53.018Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751"}
{"level":"info","ts":"2026-09-18T10:19:53.018Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751"}
{"level":"info","ts":"2026-09-18T10:19:53.018Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751"}
{"level":"info","ts":"2026-09-18T10:19:53.018Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:53.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:53.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:53.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:19:53.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:53.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:53.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:53.018Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:19:53.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:53.018Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:19:53.018Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:19:46Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:19:53Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-09-18T10:19:53.018Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"18771148-c9ff-464b-8ae7-c30861709c35","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:53.019Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"18771148-c9ff-464b-8ae7-c30861709c35","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:19:53.019Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"18771148-c9ff-464b-8ae7-c30861709c35","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-09-18T10:19:53.071Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"bf4fa393-c015-437c-b58d-b903c8401b73","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:19:53.071Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"bf4fa393-c015-437c-b58d-b903c8401b73","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:53.071Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8f8d37ee-c7c5-47a2-9031-fa1e7e922bdc","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:53.073Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-wlgxb","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:19:53.073Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751"}
{"level":"error","ts":"2026-09-18T10:19:53.073Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b450f0ea-7e9a-422a-af46-73a3f460c751","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:19:53.073Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e6a6a1a3-2d9f-4621-bca8-cc7d716b818a"}
{"level":"info","ts":"2026-09-18T10:19:53.073Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e6a6a1a3-2d9f-4621-bca8-cc7d716b818a"}
{"level":"info","ts":"2026-09-18T10:19:53.073Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e6a6a1a3-2d9f-4621-bca8-cc7d716b818a"}
{"level":"info","ts":"2026-09-18T10:19:53.073Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e6a6a1a3-2d9f-4621-bca8-cc7d716b818a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:53.073Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e6a6a1a3-2d9f-4621-bca8-cc7d716b818a","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:53.073Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e6a6a1a3-2d9f-4621-bca8-cc7d716b818a","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:53.073Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e6a6a1a3-2d9f-4621-bca8-cc7d716b818a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:19:53.074Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e6a6a1a3-2d9f-4621-bca8-cc7d716b818a","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:53.074Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e6a6a1a3-2d9f-4621-bca8-cc7d716b818a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:53.074Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e6a6a1a3-2d9f-4621-bca8-cc7d716b818a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:53.074Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e6a6a1a3-2d9f-4621-bca8-cc7d716b818a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:19:53.074Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e6a6a1a3-2d9f-4621-bca8-cc7d716b818a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:53.074Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e6a6a1a3-2d9f-4621-bca8-cc7d716b818a","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:19:53.074Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e6a6a1a3-2d9f-4621-bca8-cc7d716b818a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:19:53.074Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e6a6a1a3-2d9f-4621-bca8-cc7d716b818a","seconds":300}
{"level":"info","ts":"2026-09-18T10:19:53.078Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"75c730b4-a395-4f64-b019-a54ff6efdd60"}
{"level":"info","ts":"2026-09-18T10:19:53.078Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"75c730b4-a395-4f64-b019-a54ff6efdd60"}
{"level":"info","ts":"2026-09-18T10:19:53.078Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"75c730b4-a395-4f64-b019-a54ff6efdd60"}
{"level":"info","ts":"2026-09-18T10:19:53.078Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"75c730b4-a395-4f64-b019-a54ff6efdd60","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:53.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"75c730b4-a395-4f64-b019-a54ff6efdd60","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:53.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"75c730b4-a395-4f64-b019-a54ff6efdd60","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:53.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"75c730b4-a395-4f64-b019-a54ff6efdd60","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:19:53.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"75c730b4-a395-4f64-b019-a54ff6efdd60","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:53.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"75c730b4-a395-4f64-b019-a54ff6efdd60","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:53.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"75c730b4-a395-4f64-b019-a54ff6efdd60","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:53.079Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"75c730b4-a395-4f64-b019-a54ff6efdd60","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:19:53.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"75c730b4-a395-4f64-b019-a54ff6efdd60","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:53.079Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"75c730b4-a395-4f64-b019-a54ff6efdd60","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:19:53.079Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"75c730b4-a395-4f64-b019-a54ff6efdd60","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:19:53.079Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"75c730b4-a395-4f64-b019-a54ff6efdd60","seconds":300}
{"level":"info","ts":"2026-09-18T10:19:53.083Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8f8d37ee-c7c5-47a2-9031-fa1e7e922bdc","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:19:53.099Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8f8d37ee-c7c5-47a2-9031-fa1e7e922bdc","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:19:53.099Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8f8d37ee-c7c5-47a2-9031-fa1e7e922bdc","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:53.596Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52db9158-5ac2-4cd9-9b6a-4f1233c0c0c0"}
{"level":"info","ts":"2026-09-18T10:19:53.596Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52db9158-5ac2-4cd9-9b6a-4f1233c0c0c0"}
{"level":"info","ts":"2026-09-18T10:19:53.596Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52db9158-5ac2-4cd9-9b6a-4f1233c0c0c0"}
{"level":"info","ts":"2026-09-18T10:19:53.596Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52db9158-5ac2-4cd9-9b6a-4f1233c0c0c0","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:53.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52db9158-5ac2-4cd9-9b6a-4f1233c0c0c0","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:53.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52db9158-5ac2-4cd9-9b6a-4f1233c0c0c0","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:53.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52db9158-5ac2-4cd9-9b6a-4f1233c0c0c0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:19:53.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52db9158-5ac2-4cd9-9b6a-4f1233c0c0c0","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:53.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52db9158-5ac2-4cd9-9b6a-4f1233c0c0c0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:53.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52db9158-5ac2-4cd9-9b6a-4f1233c0c0c0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:53.596Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52db9158-5ac2-4cd9-9b6a-4f1233c0c0c0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:19:53.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52db9158-5ac2-4cd9-9b6a-4f1233c0c0c0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:53.596Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52db9158-5ac2-4cd9-9b6a-4f1233c0c0c0","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:19:53.596Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52db9158-5ac2-4cd9-9b6a-4f1233c0c0c0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:19:53.596Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52db9158-5ac2-4cd9-9b6a-4f1233c0c0c0","seconds":300}
{"level":"info","ts":"2026-09-18T10:19:53.981Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f2918e47-a65b-43a7-a88b-ea4417b0132d"}
{"level":"info","ts":"2026-09-18T10:19:53.981Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f2918e47-a65b-43a7-a88b-ea4417b0132d"}
{"level":"info","ts":"2026-09-18T10:19:53.981Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f2918e47-a65b-43a7-a88b-ea4417b0132d"}
{"level":"info","ts":"2026-09-18T10:19:53.981Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f2918e47-a65b-43a7-a88b-ea4417b0132d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:53.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f2918e47-a65b-43a7-a88b-ea4417b0132d","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:53.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f2918e47-a65b-43a7-a88b-ea4417b0132d","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:53.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f2918e47-a65b-43a7-a88b-ea4417b0132d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:19:53.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f2918e47-a65b-43a7-a88b-ea4417b0132d","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:53.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f2918e47-a65b-43a7-a88b-ea4417b0132d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:53.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f2918e47-a65b-43a7-a88b-ea4417b0132d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:53.981Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f2918e47-a65b-43a7-a88b-ea4417b0132d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:19:53.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f2918e47-a65b-43a7-a88b-ea4417b0132d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:53.981Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f2918e47-a65b-43a7-a88b-ea4417b0132d","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:19:53.982Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f2918e47-a65b-43a7-a88b-ea4417b0132d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:19:53.982Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f2918e47-a65b-43a7-a88b-ea4417b0132d","seconds":300}
{"level":"info","ts":"2026-09-18T10:19:58.110Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:58.110Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-wlgxb"}
{"level":"info","ts":"2026-09-18T10:19:58.130Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"2700bb43-87fb-4cf9-b67d-0300aa35285c","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:58.130Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"be2ce567-a644-436a-9cb7-0e3e7159f5d7","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:58.130Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"2700bb43-87fb-4cf9-b67d-0300aa35285c","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:19:58.130Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"2700bb43-87fb-4cf9-b67d-0300aa35285c","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:19:58.130Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"325bb1ec-3b23-437c-a031-d8cf8ac84dc6","namespacedName":{"name":"ray-cluster","namespace":"test-ns-wlgxb"}}
{"level":"info","ts":"2026-09-18T10:19:58.130Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"325bb1ec-3b23-437c-a031-d8cf8ac84dc6","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:58.130Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"325bb1ec-3b23-437c-a031-d8cf8ac84dc6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:58.130Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b"}
{"level":"info","ts":"2026-09-18T10:19:58.130Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b"}
{"level":"info","ts":"2026-09-18T10:19:58.130Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b"}
{"level":"info","ts":"2026-09-18T10:19:58.130Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:58.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:58.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:19:58.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:58.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","workerReplicas":2,"NumOfHosts":1,"runningPods":1,"diff":1}
{"level":"info","ts":"2026-09-18T10:19:58.131Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:19:58.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:58.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":1}
{"level":"info","ts":"2026-09-18T10:19:58.131Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b"}
{"level":"info","ts":"2026-09-18T10:19:58.131Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","nodeType":"worker","rayStartParams":{"address":"ray-cluster-head-svc.test-ns-wlgxb.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"1G"},"requests":{"cpu":"1","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:19:58.131Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","rayStartCmd":"ray start  --address=ray-cluster-head-svc.test-ns-wlgxb.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:19:58.131Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=ray-cluster-head-svc.test-ns-wlgxb.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:19:58.131Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","enabled":true}
{"level":"info","ts":"2026-09-18T10:19:58.143Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"be2ce567-a644-436a-9cb7-0e3e7159f5d7","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:19:58.147Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"325bb1ec-3b23-437c-a031-d8cf8ac84dc6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:58.154Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","name":"ray-cluster-small-group-worker-ldvkv"}
{"level":"info","ts":"2026-09-18T10:19:58.154Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:19:58.154Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:19:52Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:19:58Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-09-18T10:19:58.158Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"be2ce567-a644-436a-9cb7-0e3e7159f5d7","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:19:58.158Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"be2ce567-a644-436a-9cb7-0e3e7159f5d7","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:58.158Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"494faf6e-21d4-4d24-a5aa-f0a7e6b17947","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:58.165Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-wlgxb","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:19:58.165Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b"}
{"level":"error","ts":"2026-09-18T10:19:58.166Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a7dafd7c-7dc2-43af-bc1a-ee4933db149b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:19:58.166Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4b12896d-6f90-409f-84da-b075ba68d143"}
{"level":"info","ts":"2026-09-18T10:19:58.166Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4b12896d-6f90-409f-84da-b075ba68d143"}
{"level":"info","ts":"2026-09-18T10:19:58.166Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4b12896d-6f90-409f-84da-b075ba68d143"}
{"level":"info","ts":"2026-09-18T10:19:58.166Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4b12896d-6f90-409f-84da-b075ba68d143","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:58.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4b12896d-6f90-409f-84da-b075ba68d143","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:58.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4b12896d-6f90-409f-84da-b075ba68d143","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4b12896d-6f90-409f-84da-b075ba68d143","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:19:58.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4b12896d-6f90-409f-84da-b075ba68d143","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4b12896d-6f90-409f-84da-b075ba68d143","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4b12896d-6f90-409f-84da-b075ba68d143","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:58.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4b12896d-6f90-409f-84da-b075ba68d143","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:58.166Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4b12896d-6f90-409f-84da-b075ba68d143","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:19:58.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4b12896d-6f90-409f-84da-b075ba68d143","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:58.166Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4b12896d-6f90-409f-84da-b075ba68d143","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:19:58.166Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4b12896d-6f90-409f-84da-b075ba68d143","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:19:52Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:19:58Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-09-18T10:19:58.169Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"2700bb43-87fb-4cf9-b67d-0300aa35285c","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-wlgxb.svc","ray-cluster-head-svc.test-ns-wlgxb.svc.cluster.local"],"ipAddresses":["10.133.0.53","10.132.0.60","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:19:58.173Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"494faf6e-21d4-4d24-a5aa-f0a7e6b17947","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:19:58.185Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"7ce3376e-df7c-4cc0-9054-866757a664bf"}
{"level":"info","ts":"2026-09-18T10:19:58.185Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"7ce3376e-df7c-4cc0-9054-866757a664bf"}
{"level":"info","ts":"2026-09-18T10:19:58.185Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"7ce3376e-df7c-4cc0-9054-866757a664bf"}
{"level":"info","ts":"2026-09-18T10:19:58.185Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"7ce3376e-df7c-4cc0-9054-866757a664bf","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:58.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"7ce3376e-df7c-4cc0-9054-866757a664bf","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:58.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"7ce3376e-df7c-4cc0-9054-866757a664bf","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"7ce3376e-df7c-4cc0-9054-866757a664bf","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:19:58.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"7ce3376e-df7c-4cc0-9054-866757a664bf","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"7ce3376e-df7c-4cc0-9054-866757a664bf","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"7ce3376e-df7c-4cc0-9054-866757a664bf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:58.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"7ce3376e-df7c-4cc0-9054-866757a664bf","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:58.185Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"7ce3376e-df7c-4cc0-9054-866757a664bf","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:19:58.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"7ce3376e-df7c-4cc0-9054-866757a664bf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:58.185Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"7ce3376e-df7c-4cc0-9054-866757a664bf","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:19:58.185Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"7ce3376e-df7c-4cc0-9054-866757a664bf","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:19:58.185Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"7ce3376e-df7c-4cc0-9054-866757a664bf","seconds":300}
{"level":"info","ts":"2026-09-18T10:19:58.187Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3def5e7b-bb5d-4aaa-829a-37228bf676d1"}
{"level":"info","ts":"2026-09-18T10:19:58.187Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3def5e7b-bb5d-4aaa-829a-37228bf676d1"}
{"level":"info","ts":"2026-09-18T10:19:58.187Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3def5e7b-bb5d-4aaa-829a-37228bf676d1"}
{"level":"info","ts":"2026-09-18T10:19:58.187Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3def5e7b-bb5d-4aaa-829a-37228bf676d1","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:58.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3def5e7b-bb5d-4aaa-829a-37228bf676d1","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:58.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3def5e7b-bb5d-4aaa-829a-37228bf676d1","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3def5e7b-bb5d-4aaa-829a-37228bf676d1","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:19:58.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3def5e7b-bb5d-4aaa-829a-37228bf676d1","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3def5e7b-bb5d-4aaa-829a-37228bf676d1","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3def5e7b-bb5d-4aaa-829a-37228bf676d1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:58.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3def5e7b-bb5d-4aaa-829a-37228bf676d1","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:58.188Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3def5e7b-bb5d-4aaa-829a-37228bf676d1","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:19:58.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3def5e7b-bb5d-4aaa-829a-37228bf676d1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:58.188Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"2700bb43-87fb-4cf9-b67d-0300aa35285c","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-wlgxb.svc","ray-cluster-worker-svc.test-ns-wlgxb.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-wlgxb.svc","*.ray-cluster-worker-svc.test-ns-wlgxb.svc.cluster.local","*-worker-*.test-ns-wlgxb.svc","*-worker-*.test-ns-wlgxb.svc.cluster.local"],"ipAddresses":["10.133.0.53","10.132.0.60","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:19:58.188Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3def5e7b-bb5d-4aaa-829a-37228bf676d1","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:19:58.188Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"2700bb43-87fb-4cf9-b67d-0300aa35285c","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.60"]}
{"level":"info","ts":"2026-09-18T10:19:58.188Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"5a19ae90-7336-4e13-ac75-97ec786b4b5f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:58.188Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3def5e7b-bb5d-4aaa-829a-37228bf676d1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:19:58.188Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3def5e7b-bb5d-4aaa-829a-37228bf676d1","seconds":300}
{"level":"info","ts":"2026-09-18T10:19:58.188Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"5a19ae90-7336-4e13-ac75-97ec786b4b5f","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:19:58.188Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"5a19ae90-7336-4e13-ac75-97ec786b4b5f","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:19:58.188Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"5a19ae90-7336-4e13-ac75-97ec786b4b5f","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.60"]}
{"level":"info","ts":"2026-09-18T10:19:58.201Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"494faf6e-21d4-4d24-a5aa-f0a7e6b17947","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:19:58.201Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"494faf6e-21d4-4d24-a5aa-f0a7e6b17947","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:58.201Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"665c0d4c-d750-4fc9-bedb-3c05c3712444","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:58.226Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"65efc956-cb6f-4636-b110-0b8b27480101"}
{"level":"info","ts":"2026-09-18T10:19:58.226Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"65efc956-cb6f-4636-b110-0b8b27480101"}
{"level":"info","ts":"2026-09-18T10:19:58.226Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"65efc956-cb6f-4636-b110-0b8b27480101"}
{"level":"info","ts":"2026-09-18T10:19:58.226Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"65efc956-cb6f-4636-b110-0b8b27480101","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:58.226Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"65efc956-cb6f-4636-b110-0b8b27480101","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:58.226Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"65efc956-cb6f-4636-b110-0b8b27480101","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.226Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"65efc956-cb6f-4636-b110-0b8b27480101","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:19:58.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"65efc956-cb6f-4636-b110-0b8b27480101","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"65efc956-cb6f-4636-b110-0b8b27480101","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"65efc956-cb6f-4636-b110-0b8b27480101","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:58.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"65efc956-cb6f-4636-b110-0b8b27480101","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:58.227Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"65efc956-cb6f-4636-b110-0b8b27480101","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:19:58.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"65efc956-cb6f-4636-b110-0b8b27480101","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:58.227Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"65efc956-cb6f-4636-b110-0b8b27480101","podName":"ray-cluster-small-group-worker-ldvkv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:19:58.227Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"65efc956-cb6f-4636-b110-0b8b27480101","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:19:58.227Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"65efc956-cb6f-4636-b110-0b8b27480101","seconds":300}
{"level":"info","ts":"2026-09-18T10:19:58.227Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"665c0d4c-d750-4fc9-bedb-3c05c3712444","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:19:58.266Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"665c0d4c-d750-4fc9-bedb-3c05c3712444","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:19:58.266Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"665c0d4c-d750-4fc9-bedb-3c05c3712444","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:19:58.765Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8824333c-1b58-4002-ad34-670c3b1bc33f"}
{"level":"info","ts":"2026-09-18T10:19:58.765Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8824333c-1b58-4002-ad34-670c3b1bc33f"}
{"level":"info","ts":"2026-09-18T10:19:58.765Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8824333c-1b58-4002-ad34-670c3b1bc33f"}
{"level":"info","ts":"2026-09-18T10:19:58.765Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8824333c-1b58-4002-ad34-670c3b1bc33f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:58.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8824333c-1b58-4002-ad34-670c3b1bc33f","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:58.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8824333c-1b58-4002-ad34-670c3b1bc33f","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8824333c-1b58-4002-ad34-670c3b1bc33f","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:19:58.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8824333c-1b58-4002-ad34-670c3b1bc33f","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8824333c-1b58-4002-ad34-670c3b1bc33f","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:58.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8824333c-1b58-4002-ad34-670c3b1bc33f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:58.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8824333c-1b58-4002-ad34-670c3b1bc33f","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:58.765Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8824333c-1b58-4002-ad34-670c3b1bc33f","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:19:58.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8824333c-1b58-4002-ad34-670c3b1bc33f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:58.765Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8824333c-1b58-4002-ad34-670c3b1bc33f","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:19:58.766Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8824333c-1b58-4002-ad34-670c3b1bc33f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:19:58.766Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8824333c-1b58-4002-ad34-670c3b1bc33f","seconds":300}
{"level":"info","ts":"2026-09-18T10:19:59.002Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"007f5e00-d12b-4615-b167-6d850f82397a"}
{"level":"info","ts":"2026-09-18T10:19:59.002Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"007f5e00-d12b-4615-b167-6d850f82397a"}
{"level":"info","ts":"2026-09-18T10:19:59.002Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"007f5e00-d12b-4615-b167-6d850f82397a"}
{"level":"info","ts":"2026-09-18T10:19:59.002Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"007f5e00-d12b-4615-b167-6d850f82397a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:19:59.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"007f5e00-d12b-4615-b167-6d850f82397a","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:19:59.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"007f5e00-d12b-4615-b167-6d850f82397a","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:59.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"007f5e00-d12b-4615-b167-6d850f82397a","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:19:59.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"007f5e00-d12b-4615-b167-6d850f82397a","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:59.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"007f5e00-d12b-4615-b167-6d850f82397a","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:19:59.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"007f5e00-d12b-4615-b167-6d850f82397a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:59.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"007f5e00-d12b-4615-b167-6d850f82397a","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:19:59.002Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"007f5e00-d12b-4615-b167-6d850f82397a","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:19:59.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"007f5e00-d12b-4615-b167-6d850f82397a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:19:59.002Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"007f5e00-d12b-4615-b167-6d850f82397a","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:19:59.002Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"007f5e00-d12b-4615-b167-6d850f82397a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:19:59.002Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"007f5e00-d12b-4615-b167-6d850f82397a","seconds":300}
{"level":"info","ts":"2026-09-18T10:20:00.186Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2523be14-6300-4d9e-9c6a-59a60a905d3f"}
{"level":"info","ts":"2026-09-18T10:20:00.186Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2523be14-6300-4d9e-9c6a-59a60a905d3f"}
{"level":"info","ts":"2026-09-18T10:20:00.186Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2523be14-6300-4d9e-9c6a-59a60a905d3f"}
{"level":"info","ts":"2026-09-18T10:20:00.186Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2523be14-6300-4d9e-9c6a-59a60a905d3f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:20:00.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2523be14-6300-4d9e-9c6a-59a60a905d3f","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:20:00.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2523be14-6300-4d9e-9c6a-59a60a905d3f","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:20:00.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2523be14-6300-4d9e-9c6a-59a60a905d3f","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:20:00.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2523be14-6300-4d9e-9c6a-59a60a905d3f","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:20:00.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2523be14-6300-4d9e-9c6a-59a60a905d3f","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:20:00.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2523be14-6300-4d9e-9c6a-59a60a905d3f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:20:00.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2523be14-6300-4d9e-9c6a-59a60a905d3f","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:20:00.186Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2523be14-6300-4d9e-9c6a-59a60a905d3f","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:20:00.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2523be14-6300-4d9e-9c6a-59a60a905d3f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:20:00.186Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2523be14-6300-4d9e-9c6a-59a60a905d3f","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:20:00.186Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2523be14-6300-4d9e-9c6a-59a60a905d3f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:20:00.186Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2523be14-6300-4d9e-9c6a-59a60a905d3f","seconds":300}
{"level":"info","ts":"2026-09-18T10:20:21.682Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"3027d984-a0d5-451c-87de-7f12b576c4eb","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:20:21.682Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"3027d984-a0d5-451c-87de-7f12b576c4eb","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:20:21.682Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"3027d984-a0d5-451c-87de-7f12b576c4eb","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:20:21.682Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"3027d984-a0d5-451c-87de-7f12b576c4eb","pod":"ray-cluster-small-group-worker-ldvkv","ip":"10.132.0.61"}
{"level":"info","ts":"2026-09-18T10:20:21.702Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"3027d984-a0d5-451c-87de-7f12b576c4eb","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-wlgxb.svc","ray-cluster-head-svc.test-ns-wlgxb.svc.cluster.local"],"ipAddresses":["10.133.0.53","10.132.0.60","10.132.0.61","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:20:21.721Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"3027d984-a0d5-451c-87de-7f12b576c4eb","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-wlgxb.svc","ray-cluster-worker-svc.test-ns-wlgxb.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-wlgxb.svc","*.ray-cluster-worker-svc.test-ns-wlgxb.svc.cluster.local","*-worker-*.test-ns-wlgxb.svc","*-worker-*.test-ns-wlgxb.svc.cluster.local"],"ipAddresses":["10.133.0.53","10.132.0.60","10.132.0.61","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:20:21.721Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"3027d984-a0d5-451c-87de-7f12b576c4eb","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.60","10.132.0.61"]}
{"level":"info","ts":"2026-09-18T10:21:09.330Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"92eab282-49e0-4bdd-8617-344f606f6489"}
{"level":"info","ts":"2026-09-18T10:21:09.330Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"92eab282-49e0-4bdd-8617-344f606f6489"}
{"level":"info","ts":"2026-09-18T10:21:09.330Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"92eab282-49e0-4bdd-8617-344f606f6489"}
{"level":"info","ts":"2026-09-18T10:21:09.330Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"92eab282-49e0-4bdd-8617-344f606f6489","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:21:09.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"92eab282-49e0-4bdd-8617-344f606f6489","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:21:09.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"92eab282-49e0-4bdd-8617-344f606f6489","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:09.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"92eab282-49e0-4bdd-8617-344f606f6489","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:21:09.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"92eab282-49e0-4bdd-8617-344f606f6489","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:09.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"92eab282-49e0-4bdd-8617-344f606f6489","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:09.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"92eab282-49e0-4bdd-8617-344f606f6489","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:09.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"92eab282-49e0-4bdd-8617-344f606f6489","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:21:09.330Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"92eab282-49e0-4bdd-8617-344f606f6489","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:21:09.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"92eab282-49e0-4bdd-8617-344f606f6489","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:09.330Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"92eab282-49e0-4bdd-8617-344f606f6489","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:21:09.331Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"92eab282-49e0-4bdd-8617-344f606f6489","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:21:09.331Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"92eab282-49e0-4bdd-8617-344f606f6489","seconds":300}
{"level":"info","ts":"2026-09-18T10:21:09.356Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ab7e8daa-d950-462f-bacb-bb94c1db3775"}
{"level":"info","ts":"2026-09-18T10:21:09.356Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ab7e8daa-d950-462f-bacb-bb94c1db3775"}
{"level":"info","ts":"2026-09-18T10:21:09.356Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ab7e8daa-d950-462f-bacb-bb94c1db3775"}
{"level":"info","ts":"2026-09-18T10:21:09.356Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ab7e8daa-d950-462f-bacb-bb94c1db3775","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:21:09.356Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ab7e8daa-d950-462f-bacb-bb94c1db3775","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:21:09.356Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ab7e8daa-d950-462f-bacb-bb94c1db3775","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:09.356Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ab7e8daa-d950-462f-bacb-bb94c1db3775","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:21:09.356Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ab7e8daa-d950-462f-bacb-bb94c1db3775","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:09.356Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ab7e8daa-d950-462f-bacb-bb94c1db3775","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:09.356Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ab7e8daa-d950-462f-bacb-bb94c1db3775","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:09.356Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ab7e8daa-d950-462f-bacb-bb94c1db3775","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:21:09.356Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ab7e8daa-d950-462f-bacb-bb94c1db3775","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:21:09.356Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ab7e8daa-d950-462f-bacb-bb94c1db3775","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:09.357Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ab7e8daa-d950-462f-bacb-bb94c1db3775","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:21:09.357Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ab7e8daa-d950-462f-bacb-bb94c1db3775","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:21:09.357Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ab7e8daa-d950-462f-bacb-bb94c1db3775","seconds":300}
{"level":"info","ts":"2026-09-18T10:21:10.331Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ffef0a64-fc03-4cbb-985e-27dba55f6f9d"}
{"level":"info","ts":"2026-09-18T10:21:10.331Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ffef0a64-fc03-4cbb-985e-27dba55f6f9d"}
{"level":"info","ts":"2026-09-18T10:21:10.331Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ffef0a64-fc03-4cbb-985e-27dba55f6f9d"}
{"level":"info","ts":"2026-09-18T10:21:10.331Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ffef0a64-fc03-4cbb-985e-27dba55f6f9d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:21:10.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ffef0a64-fc03-4cbb-985e-27dba55f6f9d","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:21:10.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ffef0a64-fc03-4cbb-985e-27dba55f6f9d","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:10.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ffef0a64-fc03-4cbb-985e-27dba55f6f9d","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:21:10.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ffef0a64-fc03-4cbb-985e-27dba55f6f9d","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:10.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ffef0a64-fc03-4cbb-985e-27dba55f6f9d","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:10.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ffef0a64-fc03-4cbb-985e-27dba55f6f9d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:10.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ffef0a64-fc03-4cbb-985e-27dba55f6f9d","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:21:10.332Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ffef0a64-fc03-4cbb-985e-27dba55f6f9d","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:21:10.332Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ffef0a64-fc03-4cbb-985e-27dba55f6f9d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:10.332Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ffef0a64-fc03-4cbb-985e-27dba55f6f9d","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:21:10.332Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ffef0a64-fc03-4cbb-985e-27dba55f6f9d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:21:10.332Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ffef0a64-fc03-4cbb-985e-27dba55f6f9d","seconds":300}
{"level":"info","ts":"2026-09-18T10:21:10.353Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4f065c69-370d-4b73-bed2-1cd7c8f6d2ed"}
{"level":"info","ts":"2026-09-18T10:21:10.353Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4f065c69-370d-4b73-bed2-1cd7c8f6d2ed"}
{"level":"info","ts":"2026-09-18T10:21:10.353Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4f065c69-370d-4b73-bed2-1cd7c8f6d2ed"}
{"level":"info","ts":"2026-09-18T10:21:10.353Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4f065c69-370d-4b73-bed2-1cd7c8f6d2ed","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:21:10.354Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4f065c69-370d-4b73-bed2-1cd7c8f6d2ed","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:21:10.354Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4f065c69-370d-4b73-bed2-1cd7c8f6d2ed","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:10.354Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4f065c69-370d-4b73-bed2-1cd7c8f6d2ed","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:21:10.354Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4f065c69-370d-4b73-bed2-1cd7c8f6d2ed","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:10.354Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4f065c69-370d-4b73-bed2-1cd7c8f6d2ed","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:10.354Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4f065c69-370d-4b73-bed2-1cd7c8f6d2ed","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:10.354Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4f065c69-370d-4b73-bed2-1cd7c8f6d2ed","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:21:10.354Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4f065c69-370d-4b73-bed2-1cd7c8f6d2ed","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:21:10.354Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4f065c69-370d-4b73-bed2-1cd7c8f6d2ed","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:10.354Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4f065c69-370d-4b73-bed2-1cd7c8f6d2ed","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:21:10.354Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4f065c69-370d-4b73-bed2-1cd7c8f6d2ed","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:21:10.354Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4f065c69-370d-4b73-bed2-1cd7c8f6d2ed","seconds":300}
{"level":"info","ts":"2026-09-18T10:21:19.381Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b357fa07-3737-45da-9643-736524450666"}
{"level":"info","ts":"2026-09-18T10:21:19.381Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b357fa07-3737-45da-9643-736524450666"}
{"level":"info","ts":"2026-09-18T10:21:19.381Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b357fa07-3737-45da-9643-736524450666"}
{"level":"info","ts":"2026-09-18T10:21:19.381Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b357fa07-3737-45da-9643-736524450666","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:21:19.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b357fa07-3737-45da-9643-736524450666","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:21:19.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b357fa07-3737-45da-9643-736524450666","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:19.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b357fa07-3737-45da-9643-736524450666","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:21:19.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b357fa07-3737-45da-9643-736524450666","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:19.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b357fa07-3737-45da-9643-736524450666","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:19.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b357fa07-3737-45da-9643-736524450666","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:19.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b357fa07-3737-45da-9643-736524450666","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:21:19.381Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b357fa07-3737-45da-9643-736524450666","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:21:19.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b357fa07-3737-45da-9643-736524450666","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:19.381Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b357fa07-3737-45da-9643-736524450666","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:21:19.381Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b357fa07-3737-45da-9643-736524450666","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:21:19.381Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b357fa07-3737-45da-9643-736524450666","seconds":300}
{"level":"info","ts":"2026-09-18T10:21:19.403Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f886ef24-a1e7-4800-a402-3452582265ea"}
{"level":"info","ts":"2026-09-18T10:21:19.403Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f886ef24-a1e7-4800-a402-3452582265ea"}
{"level":"info","ts":"2026-09-18T10:21:19.403Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f886ef24-a1e7-4800-a402-3452582265ea"}
{"level":"info","ts":"2026-09-18T10:21:19.403Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f886ef24-a1e7-4800-a402-3452582265ea","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:21:19.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f886ef24-a1e7-4800-a402-3452582265ea","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:21:19.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f886ef24-a1e7-4800-a402-3452582265ea","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:19.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f886ef24-a1e7-4800-a402-3452582265ea","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:21:19.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f886ef24-a1e7-4800-a402-3452582265ea","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:19.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f886ef24-a1e7-4800-a402-3452582265ea","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:19.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f886ef24-a1e7-4800-a402-3452582265ea","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:19.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f886ef24-a1e7-4800-a402-3452582265ea","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:21:19.404Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f886ef24-a1e7-4800-a402-3452582265ea","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:21:19.404Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f886ef24-a1e7-4800-a402-3452582265ea","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:19.404Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f886ef24-a1e7-4800-a402-3452582265ea","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:21:19.404Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f886ef24-a1e7-4800-a402-3452582265ea","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:21:19.404Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f886ef24-a1e7-4800-a402-3452582265ea","seconds":300}
{"level":"info","ts":"2026-09-18T10:21:20.415Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ebc01920-1e2f-4d57-a57c-bf341e6d7496"}
{"level":"info","ts":"2026-09-18T10:21:20.415Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ebc01920-1e2f-4d57-a57c-bf341e6d7496"}
{"level":"info","ts":"2026-09-18T10:21:20.416Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ebc01920-1e2f-4d57-a57c-bf341e6d7496"}
{"level":"info","ts":"2026-09-18T10:21:20.416Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ebc01920-1e2f-4d57-a57c-bf341e6d7496","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:21:20.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ebc01920-1e2f-4d57-a57c-bf341e6d7496","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:21:20.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ebc01920-1e2f-4d57-a57c-bf341e6d7496","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:20.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ebc01920-1e2f-4d57-a57c-bf341e6d7496","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:21:20.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ebc01920-1e2f-4d57-a57c-bf341e6d7496","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:20.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ebc01920-1e2f-4d57-a57c-bf341e6d7496","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:20.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ebc01920-1e2f-4d57-a57c-bf341e6d7496","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:20.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ebc01920-1e2f-4d57-a57c-bf341e6d7496","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:21:20.416Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ebc01920-1e2f-4d57-a57c-bf341e6d7496","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:21:20.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ebc01920-1e2f-4d57-a57c-bf341e6d7496","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:20.416Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ebc01920-1e2f-4d57-a57c-bf341e6d7496","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:21:20.416Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ebc01920-1e2f-4d57-a57c-bf341e6d7496","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:19:58Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:21:20Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-09-18T10:21:20.434Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"7d0ae3a5-b355-4dfc-b26c-34da5fecb2b2","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:21:20.434Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"0e83cd89-10e3-49d9-b78d-fa42f0f31d34","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:21:20.434Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"7d0ae3a5-b355-4dfc-b26c-34da5fecb2b2","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:21:20.434Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"7d0ae3a5-b355-4dfc-b26c-34da5fecb2b2","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:21:20.434Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"7d0ae3a5-b355-4dfc-b26c-34da5fecb2b2","pod":"ray-cluster-small-group-worker-ldvkv","ip":"10.132.0.61"}
{"level":"info","ts":"2026-09-18T10:21:20.435Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"7d0ae3a5-b355-4dfc-b26c-34da5fecb2b2","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.60","10.132.0.61"]}
{"level":"info","ts":"2026-09-18T10:21:20.443Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52e21ee1-7976-400c-8c9a-ac76faab091c"}
{"level":"info","ts":"2026-09-18T10:21:20.443Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52e21ee1-7976-400c-8c9a-ac76faab091c"}
{"level":"info","ts":"2026-09-18T10:21:20.443Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52e21ee1-7976-400c-8c9a-ac76faab091c"}
{"level":"info","ts":"2026-09-18T10:21:20.443Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52e21ee1-7976-400c-8c9a-ac76faab091c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:21:20.443Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52e21ee1-7976-400c-8c9a-ac76faab091c","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:21:20.443Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52e21ee1-7976-400c-8c9a-ac76faab091c","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:20.443Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52e21ee1-7976-400c-8c9a-ac76faab091c","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:21:20.443Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52e21ee1-7976-400c-8c9a-ac76faab091c","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:20.443Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52e21ee1-7976-400c-8c9a-ac76faab091c","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:20.443Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52e21ee1-7976-400c-8c9a-ac76faab091c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:20.443Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52e21ee1-7976-400c-8c9a-ac76faab091c","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:21:20.443Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52e21ee1-7976-400c-8c9a-ac76faab091c","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:21:20.443Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52e21ee1-7976-400c-8c9a-ac76faab091c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:20.444Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52e21ee1-7976-400c-8c9a-ac76faab091c","podName":"ray-cluster-small-group-worker-9c9dl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:19:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:21:20.444Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"52e21ee1-7976-400c-8c9a-ac76faab091c","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:21:20Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:21:20Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-09-18T10:21:20.458Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"66db45ca-fd70-4dca-9e91-734aa0497476","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:21:20.458Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"66db45ca-fd70-4dca-9e91-734aa0497476","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:21:20.458Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"66db45ca-fd70-4dca-9e91-734aa0497476","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:21:20.458Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"66db45ca-fd70-4dca-9e91-734aa0497476","pod":"ray-cluster-small-group-worker-ldvkv","ip":"10.132.0.61"}
{"level":"info","ts":"2026-09-18T10:21:20.458Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"66db45ca-fd70-4dca-9e91-734aa0497476","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.60","10.132.0.61"]}
{"level":"info","ts":"2026-09-18T10:21:20.462Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"0e83cd89-10e3-49d9-b78d-fa42f0f31d34","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:21:20.476Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"0e83cd89-10e3-49d9-b78d-fa42f0f31d34","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:21:20.476Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"0e83cd89-10e3-49d9-b78d-fa42f0f31d34","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:21:20.476Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"691838d1-7de8-46d7-84d5-83af7d028991","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:21:20.487Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"691838d1-7de8-46d7-84d5-83af7d028991","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:21:20.498Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"691838d1-7de8-46d7-84d5-83af7d028991","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:21:20.498Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"691838d1-7de8-46d7-84d5-83af7d028991","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:21:21.722Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"58cd42e6-4174-43bc-941a-6060ef55bac1","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:21:21.722Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"58cd42e6-4174-43bc-941a-6060ef55bac1","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:21:21.722Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"58cd42e6-4174-43bc-941a-6060ef55bac1","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:21:21.722Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"58cd42e6-4174-43bc-941a-6060ef55bac1","pod":"ray-cluster-small-group-worker-ldvkv","ip":"10.132.0.61"}
{"level":"info","ts":"2026-09-18T10:21:21.723Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"58cd42e6-4174-43bc-941a-6060ef55bac1","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.60","10.132.0.61"]}
{"level":"info","ts":"2026-09-18T10:21:22.436Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6ff943f4-1058-467d-82c9-f69cf40debe2"}
{"level":"info","ts":"2026-09-18T10:21:22.436Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6ff943f4-1058-467d-82c9-f69cf40debe2"}
{"level":"info","ts":"2026-09-18T10:21:22.436Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6ff943f4-1058-467d-82c9-f69cf40debe2"}
{"level":"info","ts":"2026-09-18T10:21:22.436Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6ff943f4-1058-467d-82c9-f69cf40debe2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:21:22.436Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6ff943f4-1058-467d-82c9-f69cf40debe2","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:21:22.436Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6ff943f4-1058-467d-82c9-f69cf40debe2","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:22.436Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6ff943f4-1058-467d-82c9-f69cf40debe2","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:21:22.436Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6ff943f4-1058-467d-82c9-f69cf40debe2","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:22.436Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6ff943f4-1058-467d-82c9-f69cf40debe2","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:22.436Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6ff943f4-1058-467d-82c9-f69cf40debe2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:22.436Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6ff943f4-1058-467d-82c9-f69cf40debe2","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:21:22.436Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6ff943f4-1058-467d-82c9-f69cf40debe2","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:21:22.436Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6ff943f4-1058-467d-82c9-f69cf40debe2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:22.436Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6ff943f4-1058-467d-82c9-f69cf40debe2","podName":"ray-cluster-small-group-worker-ldvkv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:19:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:21:22.437Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6ff943f4-1058-467d-82c9-f69cf40debe2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:21:22.437Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6ff943f4-1058-467d-82c9-f69cf40debe2","seconds":300}
{"level":"info","ts":"2026-09-18T10:21:31.412Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"91060f42-54e7-442d-8b26-c6650d0a4a72"}
{"level":"info","ts":"2026-09-18T10:21:31.412Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"91060f42-54e7-442d-8b26-c6650d0a4a72"}
{"level":"info","ts":"2026-09-18T10:21:31.412Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"91060f42-54e7-442d-8b26-c6650d0a4a72"}
{"level":"info","ts":"2026-09-18T10:21:31.412Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"91060f42-54e7-442d-8b26-c6650d0a4a72","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:21:31.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"91060f42-54e7-442d-8b26-c6650d0a4a72","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:21:31.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"91060f42-54e7-442d-8b26-c6650d0a4a72","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:31.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"91060f42-54e7-442d-8b26-c6650d0a4a72","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:21:31.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"91060f42-54e7-442d-8b26-c6650d0a4a72","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:31.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"91060f42-54e7-442d-8b26-c6650d0a4a72","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:31.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"91060f42-54e7-442d-8b26-c6650d0a4a72","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:31.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"91060f42-54e7-442d-8b26-c6650d0a4a72","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:21:31.413Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"91060f42-54e7-442d-8b26-c6650d0a4a72","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:21:31.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"91060f42-54e7-442d-8b26-c6650d0a4a72","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:31.413Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"91060f42-54e7-442d-8b26-c6650d0a4a72","podName":"ray-cluster-small-group-worker-ldvkv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:19:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:21:31.413Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"91060f42-54e7-442d-8b26-c6650d0a4a72","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:21:20Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:21:31Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-09-18T10:21:31.425Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"12da6003-065e-4350-8ec2-6b1ccd5a0581","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:21:31.425Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"4ab53e87-4b9b-43d7-ba0a-09d02e30a6fe","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:21:31.426Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"12da6003-065e-4350-8ec2-6b1ccd5a0581","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:21:31.426Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"12da6003-065e-4350-8ec2-6b1ccd5a0581","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:21:31.426Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"12da6003-065e-4350-8ec2-6b1ccd5a0581","pod":"ray-cluster-small-group-worker-ldvkv","ip":"10.132.0.61"}
{"level":"info","ts":"2026-09-18T10:21:31.426Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"12da6003-065e-4350-8ec2-6b1ccd5a0581","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.60","10.132.0.61"]}
{"level":"info","ts":"2026-09-18T10:21:31.435Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8e7bd053-0165-4efa-883a-17297213fb4b"}
{"level":"info","ts":"2026-09-18T10:21:31.435Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8e7bd053-0165-4efa-883a-17297213fb4b"}
{"level":"info","ts":"2026-09-18T10:21:31.435Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8e7bd053-0165-4efa-883a-17297213fb4b"}
{"level":"info","ts":"2026-09-18T10:21:31.435Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8e7bd053-0165-4efa-883a-17297213fb4b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:21:31.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8e7bd053-0165-4efa-883a-17297213fb4b","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:21:31.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8e7bd053-0165-4efa-883a-17297213fb4b","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:31.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8e7bd053-0165-4efa-883a-17297213fb4b","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:21:31.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8e7bd053-0165-4efa-883a-17297213fb4b","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:31.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8e7bd053-0165-4efa-883a-17297213fb4b","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:31.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8e7bd053-0165-4efa-883a-17297213fb4b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:31.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8e7bd053-0165-4efa-883a-17297213fb4b","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:21:31.435Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8e7bd053-0165-4efa-883a-17297213fb4b","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:21:31.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8e7bd053-0165-4efa-883a-17297213fb4b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:31.435Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8e7bd053-0165-4efa-883a-17297213fb4b","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:21:31Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:21:31Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-09-18T10:21:31.447Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"4ab53e87-4b9b-43d7-ba0a-09d02e30a6fe","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:21:31.449Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a3ed4867-cf5a-4c54-9a99-660d635e1cc1","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:21:31.449Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a3ed4867-cf5a-4c54-9a99-660d635e1cc1","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:21:31.449Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a3ed4867-cf5a-4c54-9a99-660d635e1cc1","pod":"ray-cluster-small-group-worker-ldvkv","ip":"10.132.0.61"}
{"level":"info","ts":"2026-09-18T10:21:31.449Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a3ed4867-cf5a-4c54-9a99-660d635e1cc1","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:21:31.449Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a3ed4867-cf5a-4c54-9a99-660d635e1cc1","rayCluster":"ray-cluster","podIPs":["10.132.0.60","10.132.0.61","10.133.0.53"]}
{"level":"info","ts":"2026-09-18T10:21:31.459Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"4ab53e87-4b9b-43d7-ba0a-09d02e30a6fe","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:21:31.459Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"4ab53e87-4b9b-43d7-ba0a-09d02e30a6fe","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:21:31.459Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"523a601c-e73d-4395-bd66-f02bb5669bfe","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:21:31.468Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"523a601c-e73d-4395-bd66-f02bb5669bfe","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:21:31.477Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"523a601c-e73d-4395-bd66-f02bb5669bfe","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:21:31.477Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"523a601c-e73d-4395-bd66-f02bb5669bfe","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:21:33.426Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"39a837bf-29c2-4670-bca5-f7f50d8b6755"}
{"level":"info","ts":"2026-09-18T10:21:33.426Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"39a837bf-29c2-4670-bca5-f7f50d8b6755"}
{"level":"info","ts":"2026-09-18T10:21:33.426Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"39a837bf-29c2-4670-bca5-f7f50d8b6755"}
{"level":"info","ts":"2026-09-18T10:21:33.426Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"39a837bf-29c2-4670-bca5-f7f50d8b6755","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:21:33.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"39a837bf-29c2-4670-bca5-f7f50d8b6755","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:21:33.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"39a837bf-29c2-4670-bca5-f7f50d8b6755","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:33.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"39a837bf-29c2-4670-bca5-f7f50d8b6755","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-09-18T10:21:33.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"39a837bf-29c2-4670-bca5-f7f50d8b6755","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:33.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"39a837bf-29c2-4670-bca5-f7f50d8b6755","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:21:33.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"39a837bf-29c2-4670-bca5-f7f50d8b6755","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:33.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"39a837bf-29c2-4670-bca5-f7f50d8b6755","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-09-18T10:21:33.426Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"39a837bf-29c2-4670-bca5-f7f50d8b6755","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-09-18T10:21:33.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"39a837bf-29c2-4670-bca5-f7f50d8b6755","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:21:33.426Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"39a837bf-29c2-4670-bca5-f7f50d8b6755","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:21:33.426Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"39a837bf-29c2-4670-bca5-f7f50d8b6755","seconds":300}
{"level":"info","ts":"2026-09-18T10:22:21.724Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9be0a45e-bfd9-4210-bc1a-9f0c8879b69a","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:21.724Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9be0a45e-bfd9-4210-bc1a-9f0c8879b69a","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:22:21.724Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9be0a45e-bfd9-4210-bc1a-9f0c8879b69a","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:22:21.724Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9be0a45e-bfd9-4210-bc1a-9f0c8879b69a","pod":"ray-cluster-small-group-worker-ldvkv","ip":"10.132.0.61"}
{"level":"info","ts":"2026-09-18T10:22:21.724Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9be0a45e-bfd9-4210-bc1a-9f0c8879b69a","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.60","10.132.0.61"]}
{"level":"info","ts":"2026-09-18T10:22:30.538Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:30.538Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-wlgxb"}
{"level":"info","ts":"2026-09-18T10:22:30.554Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c2ca4a83-ef81-43b5-9205-3e9c746b1448","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:30.554Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b"}
{"level":"info","ts":"2026-09-18T10:22:30.554Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b"}
{"level":"info","ts":"2026-09-18T10:22:30.554Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b"}
{"level":"info","ts":"2026-09-18T10:22:30.554Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:22:30.554Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c2ca4a83-ef81-43b5-9205-3e9c746b1448","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:22:30.554Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c2ca4a83-ef81-43b5-9205-3e9c746b1448","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:22:30.554Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c2ca4a83-ef81-43b5-9205-3e9c746b1448","pod":"ray-cluster-small-group-worker-ldvkv","ip":"10.132.0.61"}
{"level":"info","ts":"2026-09-18T10:22:30.554Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c2ca4a83-ef81-43b5-9205-3e9c746b1448","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.60","10.132.0.61"]}
{"level":"info","ts":"2026-09-18T10:22:30.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:22:30.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:22:30.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:22:30.554Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"400c0062-36c9-4eaf-802c-cd4715718e9d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:30.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:22:30.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b","worker Pod":"ray-cluster-small-group-worker-ldvkv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-ldvkv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:22:30.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:22:30.555Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b","namespace":"test-ns-wlgxb","name":"ray-cluster-small-group-worker-ldvkv"}
{"level":"info","ts":"2026-09-18T10:22:30.555Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"debd4c7e-23ab-4ff6-b183-8affbfeeed74","namespacedName":{"name":"ray-cluster","namespace":"test-ns-wlgxb"}}
{"level":"info","ts":"2026-09-18T10:22:30.555Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"debd4c7e-23ab-4ff6-b183-8affbfeeed74","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:30.555Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"debd4c7e-23ab-4ff6-b183-8affbfeeed74","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:30.584Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:22:30.584Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:22:30.584Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:22:30.584Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"debd4c7e-23ab-4ff6-b183-8affbfeeed74","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:30.584Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:21:31Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:22:30Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-09-18T10:22:30.587Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"be96ccff-04fe-4fb3-920e-1413da84ac75","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:30.587Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"be96ccff-04fe-4fb3-920e-1413da84ac75","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:22:30.587Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"be96ccff-04fe-4fb3-920e-1413da84ac75","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:22:30.587Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"be96ccff-04fe-4fb3-920e-1413da84ac75","pod":"ray-cluster-small-group-worker-ldvkv","ip":"10.132.0.61"}
{"level":"info","ts":"2026-09-18T10:22:30.587Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"be96ccff-04fe-4fb3-920e-1413da84ac75","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.60","10.132.0.61"]}
{"level":"info","ts":"2026-09-18T10:22:30.600Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"400c0062-36c9-4eaf-802c-cd4715718e9d","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:22:30.602Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-wlgxb","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:22:30.602Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b"}
{"level":"error","ts":"2026-09-18T10:22:30.602Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"43f56fbd-7fec-4739-9f0e-c065ac04b89b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:22:30.602Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a1e3c858-2656-464f-9433-cc287fcc3372"}
{"level":"info","ts":"2026-09-18T10:22:30.602Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a1e3c858-2656-464f-9433-cc287fcc3372"}
{"level":"info","ts":"2026-09-18T10:22:30.602Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a1e3c858-2656-464f-9433-cc287fcc3372"}
{"level":"info","ts":"2026-09-18T10:22:30.602Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a1e3c858-2656-464f-9433-cc287fcc3372","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:22:30.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a1e3c858-2656-464f-9433-cc287fcc3372","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:22:30.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a1e3c858-2656-464f-9433-cc287fcc3372","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:22:30.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a1e3c858-2656-464f-9433-cc287fcc3372","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:22:30.602Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a1e3c858-2656-464f-9433-cc287fcc3372","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:21:31Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:22:30Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-09-18T10:22:30.623Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"400c0062-36c9-4eaf-802c-cd4715718e9d","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:22:30.623Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"400c0062-36c9-4eaf-802c-cd4715718e9d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:30.623Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"04aefab0-1c7b-4e48-bc88-c723a4d7c449","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:30.623Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"f6cc5650-4908-4e45-af6e-9a7ee30f3430","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:30.623Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"f6cc5650-4908-4e45-af6e-9a7ee30f3430","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:22:30.623Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"f6cc5650-4908-4e45-af6e-9a7ee30f3430","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:22:30.623Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"f6cc5650-4908-4e45-af6e-9a7ee30f3430","pod":"ray-cluster-small-group-worker-ldvkv","ip":"10.132.0.61"}
{"level":"info","ts":"2026-09-18T10:22:30.623Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"f6cc5650-4908-4e45-af6e-9a7ee30f3430","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.60","10.132.0.61"]}
{"level":"info","ts":"2026-09-18T10:22:30.624Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6512e2e9-2180-4e32-9f74-d999c5e4924b"}
{"level":"info","ts":"2026-09-18T10:22:30.624Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6512e2e9-2180-4e32-9f74-d999c5e4924b"}
{"level":"info","ts":"2026-09-18T10:22:30.624Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6512e2e9-2180-4e32-9f74-d999c5e4924b"}
{"level":"info","ts":"2026-09-18T10:22:30.624Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6512e2e9-2180-4e32-9f74-d999c5e4924b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:22:30.624Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6512e2e9-2180-4e32-9f74-d999c5e4924b","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:22:30.624Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6512e2e9-2180-4e32-9f74-d999c5e4924b","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:22:30.624Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6512e2e9-2180-4e32-9f74-d999c5e4924b","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:22:30.624Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6512e2e9-2180-4e32-9f74-d999c5e4924b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:22:30.624Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"6512e2e9-2180-4e32-9f74-d999c5e4924b","seconds":300}
{"level":"info","ts":"2026-09-18T10:22:30.635Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"103b0513-d299-480e-8a77-6f76eacb9686"}
{"level":"info","ts":"2026-09-18T10:22:30.635Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"103b0513-d299-480e-8a77-6f76eacb9686"}
{"level":"info","ts":"2026-09-18T10:22:30.635Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"103b0513-d299-480e-8a77-6f76eacb9686"}
{"level":"info","ts":"2026-09-18T10:22:30.635Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"103b0513-d299-480e-8a77-6f76eacb9686","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:22:30.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"103b0513-d299-480e-8a77-6f76eacb9686","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:22:30.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"103b0513-d299-480e-8a77-6f76eacb9686","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:22:30.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"103b0513-d299-480e-8a77-6f76eacb9686","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:22:30.635Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"103b0513-d299-480e-8a77-6f76eacb9686","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:22:30.635Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"103b0513-d299-480e-8a77-6f76eacb9686","seconds":300}
{"level":"info","ts":"2026-09-18T10:22:30.647Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"04aefab0-1c7b-4e48-bc88-c723a4d7c449","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:22:30.663Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"04aefab0-1c7b-4e48-bc88-c723a4d7c449","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:22:30.663Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"04aefab0-1c7b-4e48-bc88-c723a4d7c449","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:30.663Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"16cb8b35-8de9-473f-beae-2547ab3fa183","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:30.681Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"16cb8b35-8de9-473f-beae-2547ab3fa183","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:22:30.691Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"16cb8b35-8de9-473f-beae-2547ab3fa183","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:22:30.691Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"16cb8b35-8de9-473f-beae-2547ab3fa183","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:31.056Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b795a004-e2e7-4b81-9c4f-ee2cb277e83a"}
{"level":"info","ts":"2026-09-18T10:22:31.056Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b795a004-e2e7-4b81-9c4f-ee2cb277e83a"}
{"level":"info","ts":"2026-09-18T10:22:31.056Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b795a004-e2e7-4b81-9c4f-ee2cb277e83a"}
{"level":"info","ts":"2026-09-18T10:22:31.056Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b795a004-e2e7-4b81-9c4f-ee2cb277e83a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:22:31.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b795a004-e2e7-4b81-9c4f-ee2cb277e83a","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:22:31.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b795a004-e2e7-4b81-9c4f-ee2cb277e83a","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:22:31.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b795a004-e2e7-4b81-9c4f-ee2cb277e83a","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:22:31.056Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b795a004-e2e7-4b81-9c4f-ee2cb277e83a","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:22:30Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:22:31Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-09-18T10:22:31.069Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"71d79e41-5ac4-46b5-b5ac-f6d4bae29fd5","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:31.069Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"13eb7002-7396-4c1f-906c-c87053396d14","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:31.069Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"13eb7002-7396-4c1f-906c-c87053396d14","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:22:31.069Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"13eb7002-7396-4c1f-906c-c87053396d14","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:22:31.069Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"13eb7002-7396-4c1f-906c-c87053396d14","pod":"ray-cluster-small-group-worker-ldvkv","ip":"10.132.0.61"}
{"level":"info","ts":"2026-09-18T10:22:31.069Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"13eb7002-7396-4c1f-906c-c87053396d14","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.60","10.132.0.61"]}
{"level":"info","ts":"2026-09-18T10:22:31.082Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"71d79e41-5ac4-46b5-b5ac-f6d4bae29fd5","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:22:31.094Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"71d79e41-5ac4-46b5-b5ac-f6d4bae29fd5","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:22:31.094Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"71d79e41-5ac4-46b5-b5ac-f6d4bae29fd5","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:22:31.709Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b1aab82b-f410-4541-84f8-3e9cca0d18f8"}
{"level":"info","ts":"2026-09-18T10:22:31.709Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b1aab82b-f410-4541-84f8-3e9cca0d18f8"}
{"level":"info","ts":"2026-09-18T10:22:31.709Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b1aab82b-f410-4541-84f8-3e9cca0d18f8"}
{"level":"info","ts":"2026-09-18T10:22:31.709Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b1aab82b-f410-4541-84f8-3e9cca0d18f8","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:22:31.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b1aab82b-f410-4541-84f8-3e9cca0d18f8","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:22:31.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b1aab82b-f410-4541-84f8-3e9cca0d18f8","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:22:31.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b1aab82b-f410-4541-84f8-3e9cca0d18f8","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:22:31.710Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b1aab82b-f410-4541-84f8-3e9cca0d18f8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:22:31.710Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"b1aab82b-f410-4541-84f8-3e9cca0d18f8","seconds":300}
{"level":"info","ts":"2026-09-18T10:22:31.713Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"807fe224-e5b0-4c8a-b5f4-64bec758f07a"}
{"level":"info","ts":"2026-09-18T10:22:31.713Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"807fe224-e5b0-4c8a-b5f4-64bec758f07a"}
{"level":"info","ts":"2026-09-18T10:22:31.713Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"807fe224-e5b0-4c8a-b5f4-64bec758f07a"}
{"level":"info","ts":"2026-09-18T10:22:31.713Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"807fe224-e5b0-4c8a-b5f4-64bec758f07a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:22:31.713Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"807fe224-e5b0-4c8a-b5f4-64bec758f07a","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:22:31.713Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"807fe224-e5b0-4c8a-b5f4-64bec758f07a","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:22:31.713Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"807fe224-e5b0-4c8a-b5f4-64bec758f07a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:22:31.713Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"807fe224-e5b0-4c8a-b5f4-64bec758f07a","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:22:31.713Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"807fe224-e5b0-4c8a-b5f4-64bec758f07a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:22:31.713Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"807fe224-e5b0-4c8a-b5f4-64bec758f07a","namespace":"test-ns-wlgxb","name":"ray-cluster-small-group-worker-ldvkv"}
{"level":"info","ts":"2026-09-18T10:22:31.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"807fe224-e5b0-4c8a-b5f4-64bec758f07a","The worker Pod has already been deleted":"ray-cluster-small-group-worker-ldvkv"}
{"level":"info","ts":"2026-09-18T10:22:31.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"807fe224-e5b0-4c8a-b5f4-64bec758f07a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:22:31.716Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"807fe224-e5b0-4c8a-b5f4-64bec758f07a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:22:31.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"807fe224-e5b0-4c8a-b5f4-64bec758f07a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:22:31.717Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"807fe224-e5b0-4c8a-b5f4-64bec758f07a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:22:31.717Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"807fe224-e5b0-4c8a-b5f4-64bec758f07a","seconds":300}
{"level":"info","ts":"2026-09-18T10:22:32.625Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a5ce2caa-9a2e-4bd4-9b1d-572c505e344c"}
{"level":"info","ts":"2026-09-18T10:22:32.625Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a5ce2caa-9a2e-4bd4-9b1d-572c505e344c"}
{"level":"info","ts":"2026-09-18T10:22:32.625Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a5ce2caa-9a2e-4bd4-9b1d-572c505e344c"}
{"level":"info","ts":"2026-09-18T10:22:32.625Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a5ce2caa-9a2e-4bd4-9b1d-572c505e344c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:22:32.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a5ce2caa-9a2e-4bd4-9b1d-572c505e344c","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:22:32.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a5ce2caa-9a2e-4bd4-9b1d-572c505e344c","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:22:32.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a5ce2caa-9a2e-4bd4-9b1d-572c505e344c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-09-18T10:22:32.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a5ce2caa-9a2e-4bd4-9b1d-572c505e344c","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:22:32.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a5ce2caa-9a2e-4bd4-9b1d-572c505e344c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:22:32.625Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a5ce2caa-9a2e-4bd4-9b1d-572c505e344c","namespace":"test-ns-wlgxb","name":"ray-cluster-small-group-worker-ldvkv"}
{"level":"info","ts":"2026-09-18T10:22:32.629Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a5ce2caa-9a2e-4bd4-9b1d-572c505e344c","The worker Pod has already been deleted":"ray-cluster-small-group-worker-ldvkv"}
{"level":"info","ts":"2026-09-18T10:22:32.629Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a5ce2caa-9a2e-4bd4-9b1d-572c505e344c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:22:32.629Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a5ce2caa-9a2e-4bd4-9b1d-572c505e344c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:22:32.629Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a5ce2caa-9a2e-4bd4-9b1d-572c505e344c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:22:32.629Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a5ce2caa-9a2e-4bd4-9b1d-572c505e344c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:22:32.629Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a5ce2caa-9a2e-4bd4-9b1d-572c505e344c","seconds":300}
{"level":"info","ts":"2026-09-18T10:23:21.725Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8fa31a73-f92c-4ede-a6c4-49528712da26","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:21.725Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8fa31a73-f92c-4ede-a6c4-49528712da26","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:23:21.725Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8fa31a73-f92c-4ede-a6c4-49528712da26","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:23:21.751Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8fa31a73-f92c-4ede-a6c4-49528712da26","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-wlgxb.svc","ray-cluster-head-svc.test-ns-wlgxb.svc.cluster.local"],"ipAddresses":["10.133.0.53","10.132.0.60","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:23:21.762Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8fa31a73-f92c-4ede-a6c4-49528712da26","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-wlgxb.svc","ray-cluster-worker-svc.test-ns-wlgxb.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-wlgxb.svc","*.ray-cluster-worker-svc.test-ns-wlgxb.svc.cluster.local","*-worker-*.test-ns-wlgxb.svc","*-worker-*.test-ns-wlgxb.svc.cluster.local"],"ipAddresses":["10.133.0.53","10.132.0.60","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:23:21.762Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8fa31a73-f92c-4ede-a6c4-49528712da26","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.60"]}
{"level":"info","ts":"2026-09-18T10:23:41.583Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:41.583Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-wlgxb"}
{"level":"info","ts":"2026-09-18T10:23:41.594Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"071a6412-aa66-47e6-8bf6-488778e3a6cc","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:41.594Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c65dc116-f408-4233-9ad8-d738b908205e","namespacedName":{"name":"ray-cluster","namespace":"test-ns-wlgxb"}}
{"level":"info","ts":"2026-09-18T10:23:41.595Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c65dc116-f408-4233-9ad8-d738b908205e","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:41.595Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c65dc116-f408-4233-9ad8-d738b908205e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:41.595Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"071a6412-aa66-47e6-8bf6-488778e3a6cc","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:23:41.595Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"071a6412-aa66-47e6-8bf6-488778e3a6cc","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:23:41.595Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"071a6412-aa66-47e6-8bf6-488778e3a6cc","rayCluster":"ray-cluster","podIPs":["10.132.0.60","10.133.0.53"]}
{"level":"info","ts":"2026-09-18T10:23:41.595Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3"}
{"level":"info","ts":"2026-09-18T10:23:41.595Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3"}
{"level":"info","ts":"2026-09-18T10:23:41.595Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3"}
{"level":"info","ts":"2026-09-18T10:23:41.595Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:41.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:41.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:41.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:23:41.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3","worker Pod":"ray-cluster-small-group-worker-9c9dl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-9c9dl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:41.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:41.595Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3","namespace":"test-ns-wlgxb","name":"ray-cluster-small-group-worker-9c9dl"}
{"level":"info","ts":"2026-09-18T10:23:41.595Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8dfcd466-5a2e-4e8b-bfad-da832894533d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:41.605Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:41.605Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:23:41.605Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:41.605Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:22:31Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:41Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-09-18T10:23:41.608Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"106d8623-0925-4c37-819d-6e41f5b57c00","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:41.608Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"106d8623-0925-4c37-819d-6e41f5b57c00","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:23:41.608Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"106d8623-0925-4c37-819d-6e41f5b57c00","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:23:41.608Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"106d8623-0925-4c37-819d-6e41f5b57c00","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.60"]}
{"level":"info","ts":"2026-09-18T10:23:41.608Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c65dc116-f408-4233-9ad8-d738b908205e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:41.613Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8dfcd466-5a2e-4e8b-bfad-da832894533d","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:23:41.614Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-wlgxb","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:23:41.614Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3"}
{"level":"error","ts":"2026-09-18T10:23:41.614Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"4003189b-a3bf-4054-8701-ebaeba5ac5a3","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:41.614Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e644610d-47dc-47a5-aad4-74e2d59c99b6"}
{"level":"info","ts":"2026-09-18T10:23:41.614Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e644610d-47dc-47a5-aad4-74e2d59c99b6"}
{"level":"info","ts":"2026-09-18T10:23:41.614Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e644610d-47dc-47a5-aad4-74e2d59c99b6"}
{"level":"info","ts":"2026-09-18T10:23:41.614Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e644610d-47dc-47a5-aad4-74e2d59c99b6","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:41.614Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e644610d-47dc-47a5-aad4-74e2d59c99b6","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:41.614Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e644610d-47dc-47a5-aad4-74e2d59c99b6","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:41.614Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e644610d-47dc-47a5-aad4-74e2d59c99b6","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:23:41.614Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e644610d-47dc-47a5-aad4-74e2d59c99b6","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:22:31Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:41Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-09-18T10:23:41.624Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8dfcd466-5a2e-4e8b-bfad-da832894533d","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:23:41.624Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8dfcd466-5a2e-4e8b-bfad-da832894533d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:41.624Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"3755d2f3-47f5-4f27-aaa1-8edd0181124f","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:41.625Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8664cbd2-1fb8-4f8b-ad1b-bda089611484"}
{"level":"info","ts":"2026-09-18T10:23:41.625Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8664cbd2-1fb8-4f8b-ad1b-bda089611484"}
{"level":"info","ts":"2026-09-18T10:23:41.625Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8664cbd2-1fb8-4f8b-ad1b-bda089611484"}
{"level":"info","ts":"2026-09-18T10:23:41.625Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8664cbd2-1fb8-4f8b-ad1b-bda089611484","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:41.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8664cbd2-1fb8-4f8b-ad1b-bda089611484","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:41.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8664cbd2-1fb8-4f8b-ad1b-bda089611484","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:41.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8664cbd2-1fb8-4f8b-ad1b-bda089611484","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:23:41.625Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8664cbd2-1fb8-4f8b-ad1b-bda089611484","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:22:31Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:41Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-09-18T10:23:41.666Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"09873255-d5c6-4ca8-a02c-abaa39a9c48e","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:41.666Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"09873255-d5c6-4ca8-a02c-abaa39a9c48e","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:23:41.666Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"09873255-d5c6-4ca8-a02c-abaa39a9c48e","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:23:41.666Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"09873255-d5c6-4ca8-a02c-abaa39a9c48e","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.60"]}
{"level":"info","ts":"2026-09-18T10:23:41.671Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"3755d2f3-47f5-4f27-aaa1-8edd0181124f","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:23:41.672Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8664cbd2-1fb8-4f8b-ad1b-bda089611484","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-wlgxb","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:23:41.672Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8664cbd2-1fb8-4f8b-ad1b-bda089611484"}
{"level":"error","ts":"2026-09-18T10:23:41.672Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8664cbd2-1fb8-4f8b-ad1b-bda089611484","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:41.672Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"45f25a5c-1134-445f-a10b-d3342b745e95"}
{"level":"info","ts":"2026-09-18T10:23:41.672Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"45f25a5c-1134-445f-a10b-d3342b745e95"}
{"level":"info","ts":"2026-09-18T10:23:41.672Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"45f25a5c-1134-445f-a10b-d3342b745e95"}
{"level":"info","ts":"2026-09-18T10:23:41.672Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"45f25a5c-1134-445f-a10b-d3342b745e95","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:41.672Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"45f25a5c-1134-445f-a10b-d3342b745e95","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:41.672Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"45f25a5c-1134-445f-a10b-d3342b745e95","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:41.672Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"45f25a5c-1134-445f-a10b-d3342b745e95","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:23:41.673Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"45f25a5c-1134-445f-a10b-d3342b745e95","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:23:41.673Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"45f25a5c-1134-445f-a10b-d3342b745e95","seconds":300}
{"level":"info","ts":"2026-09-18T10:23:41.677Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ed20e80a-fbc1-4290-8a3b-4d9acddcacf7"}
{"level":"info","ts":"2026-09-18T10:23:41.677Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ed20e80a-fbc1-4290-8a3b-4d9acddcacf7"}
{"level":"info","ts":"2026-09-18T10:23:41.677Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ed20e80a-fbc1-4290-8a3b-4d9acddcacf7"}
{"level":"info","ts":"2026-09-18T10:23:41.677Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ed20e80a-fbc1-4290-8a3b-4d9acddcacf7","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:41.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ed20e80a-fbc1-4290-8a3b-4d9acddcacf7","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:41.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ed20e80a-fbc1-4290-8a3b-4d9acddcacf7","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:41.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ed20e80a-fbc1-4290-8a3b-4d9acddcacf7","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:23:41.678Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ed20e80a-fbc1-4290-8a3b-4d9acddcacf7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:23:41.678Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ed20e80a-fbc1-4290-8a3b-4d9acddcacf7","seconds":300}
{"level":"info","ts":"2026-09-18T10:23:41.679Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"3755d2f3-47f5-4f27-aaa1-8edd0181124f","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:23:41.679Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"3755d2f3-47f5-4f27-aaa1-8edd0181124f","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:41.679Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"bdad4b04-1aa0-48c5-8947-46ce428fd7c7","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:41.689Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"bdad4b04-1aa0-48c5-8947-46ce428fd7c7","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:23:41.700Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"bdad4b04-1aa0-48c5-8947-46ce428fd7c7","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:23:41.700Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"bdad4b04-1aa0-48c5-8947-46ce428fd7c7","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:42.082Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fa5e85f5-9ab6-496a-9c0d-53abd2447420"}
{"level":"info","ts":"2026-09-18T10:23:42.082Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fa5e85f5-9ab6-496a-9c0d-53abd2447420"}
{"level":"info","ts":"2026-09-18T10:23:42.082Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fa5e85f5-9ab6-496a-9c0d-53abd2447420"}
{"level":"info","ts":"2026-09-18T10:23:42.082Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fa5e85f5-9ab6-496a-9c0d-53abd2447420","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:42.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fa5e85f5-9ab6-496a-9c0d-53abd2447420","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:42.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fa5e85f5-9ab6-496a-9c0d-53abd2447420","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:42.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fa5e85f5-9ab6-496a-9c0d-53abd2447420","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:23:42.082Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fa5e85f5-9ab6-496a-9c0d-53abd2447420","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:41Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:42Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-09-18T10:23:42.095Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ea983b5f-9e02-45e3-bf26-34d6cd7b27b3","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:42.095Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8613a97a-534a-4a62-9f7a-6becf9e67c7d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:42.095Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ea983b5f-9e02-45e3-bf26-34d6cd7b27b3","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:23:42.095Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ea983b5f-9e02-45e3-bf26-34d6cd7b27b3","pod":"ray-cluster-small-group-worker-9c9dl","ip":"10.132.0.60"}
{"level":"info","ts":"2026-09-18T10:23:42.095Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ea983b5f-9e02-45e3-bf26-34d6cd7b27b3","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.60"]}
{"level":"info","ts":"2026-09-18T10:23:42.109Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8613a97a-534a-4a62-9f7a-6becf9e67c7d","name":"ray-cluster-head"}
{"level":"info","ts":"2026-09-18T10:23:42.120Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8613a97a-534a-4a62-9f7a-6becf9e67c7d","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-09-18T10:23:42.120Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8613a97a-534a-4a62-9f7a-6becf9e67c7d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:23:43.033Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64fa4f8a-4e12-4daa-be64-df7687678469"}
{"level":"info","ts":"2026-09-18T10:23:43.034Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64fa4f8a-4e12-4daa-be64-df7687678469"}
{"level":"info","ts":"2026-09-18T10:23:43.034Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64fa4f8a-4e12-4daa-be64-df7687678469"}
{"level":"info","ts":"2026-09-18T10:23:43.034Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64fa4f8a-4e12-4daa-be64-df7687678469","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:43.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64fa4f8a-4e12-4daa-be64-df7687678469","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:43.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64fa4f8a-4e12-4daa-be64-df7687678469","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:43.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64fa4f8a-4e12-4daa-be64-df7687678469","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-09-18T10:23:43.034Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64fa4f8a-4e12-4daa-be64-df7687678469","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:23:43.034Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"64fa4f8a-4e12-4daa-be64-df7687678469","seconds":300}
{"level":"info","ts":"2026-09-18T10:23:43.037Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a07a108a-7b9b-49d4-a4fc-4b6c60b5d8a3"}
{"level":"info","ts":"2026-09-18T10:23:43.037Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a07a108a-7b9b-49d4-a4fc-4b6c60b5d8a3"}
{"level":"info","ts":"2026-09-18T10:23:43.037Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a07a108a-7b9b-49d4-a4fc-4b6c60b5d8a3"}
{"level":"info","ts":"2026-09-18T10:23:43.037Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a07a108a-7b9b-49d4-a4fc-4b6c60b5d8a3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:43.037Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a07a108a-7b9b-49d4-a4fc-4b6c60b5d8a3","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:43.037Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a07a108a-7b9b-49d4-a4fc-4b6c60b5d8a3","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:43.037Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a07a108a-7b9b-49d4-a4fc-4b6c60b5d8a3","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:23:43.037Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a07a108a-7b9b-49d4-a4fc-4b6c60b5d8a3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:43.037Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a07a108a-7b9b-49d4-a4fc-4b6c60b5d8a3","namespace":"test-ns-wlgxb","name":"ray-cluster-small-group-worker-9c9dl"}
{"level":"info","ts":"2026-09-18T10:23:43.042Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a07a108a-7b9b-49d4-a4fc-4b6c60b5d8a3","The worker Pod has already been deleted":"ray-cluster-small-group-worker-9c9dl"}
{"level":"info","ts":"2026-09-18T10:23:43.042Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a07a108a-7b9b-49d4-a4fc-4b6c60b5d8a3","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:43.042Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a07a108a-7b9b-49d4-a4fc-4b6c60b5d8a3","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:23:43.042Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a07a108a-7b9b-49d4-a4fc-4b6c60b5d8a3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:43.042Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a07a108a-7b9b-49d4-a4fc-4b6c60b5d8a3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:23:43.042Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"a07a108a-7b9b-49d4-a4fc-4b6c60b5d8a3","seconds":300}
{"level":"info","ts":"2026-09-18T10:23:44.024Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.024Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth-token","namespace":"test-ns-xd9fs"}
{"level":"info","ts":"2026-09-18T10:23:44.095Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"081f2054-0ad5-4c59-9ebf-bc505d26b6b7"}
{"level":"info","ts":"2026-09-18T10:23:44.095Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"081f2054-0ad5-4c59-9ebf-bc505d26b6b7"}
{"level":"info","ts":"2026-09-18T10:23:44.095Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"081f2054-0ad5-4c59-9ebf-bc505d26b6b7"}
{"level":"info","ts":"2026-09-18T10:23:44.095Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"081f2054-0ad5-4c59-9ebf-bc505d26b6b7","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:44.095Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"081f2054-0ad5-4c59-9ebf-bc505d26b6b7","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:44.095Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"081f2054-0ad5-4c59-9ebf-bc505d26b6b7","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:44.095Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"081f2054-0ad5-4c59-9ebf-bc505d26b6b7","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:23:44.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"081f2054-0ad5-4c59-9ebf-bc505d26b6b7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:44.096Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"081f2054-0ad5-4c59-9ebf-bc505d26b6b7","namespace":"test-ns-wlgxb","name":"ray-cluster-small-group-worker-9c9dl"}
{"level":"info","ts":"2026-09-18T10:23:44.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"081f2054-0ad5-4c59-9ebf-bc505d26b6b7","The worker Pod has already been deleted":"ray-cluster-small-group-worker-9c9dl"}
{"level":"info","ts":"2026-09-18T10:23:44.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"081f2054-0ad5-4c59-9ebf-bc505d26b6b7","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:44.100Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"081f2054-0ad5-4c59-9ebf-bc505d26b6b7","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:23:44.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"081f2054-0ad5-4c59-9ebf-bc505d26b6b7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:44.100Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"081f2054-0ad5-4c59-9ebf-bc505d26b6b7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:23:44.100Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"081f2054-0ad5-4c59-9ebf-bc505d26b6b7","seconds":300}
{"level":"info","ts":"2026-09-18T10:23:44.134Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"1dc54e47-17cd-4991-85ec-22344798b40f","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.134Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"21b76109-fae2-4832-8c90-cc193cc87449","rayCluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.134Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"21b76109-fae2-4832-8c90-cc193cc87449","rayCluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.135Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"04d6b012-cca2-47ba-a441-a0da4cbea4d2","namespacedName":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"}}
{"level":"info","ts":"2026-09-18T10:23:44.135Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"04d6b012-cca2-47ba-a441-a0da4cbea4d2","mode":"IntegratedOAuth","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.135Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"04d6b012-cca2-47ba-a441-a0da4cbea4d2","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.135Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"04d6b012-cca2-47ba-a441-a0da4cbea4d2","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.135Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8d1d0ab-f6af-4664-80c0-96bcee7b90ef"}
{"level":"info","ts":"2026-09-18T10:23:44.135Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8d1d0ab-f6af-4664-80c0-96bcee7b90ef"}
{"level":"info","ts":"2026-09-18T10:23:44.135Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8d1d0ab-f6af-4664-80c0-96bcee7b90ef"}
{"level":"info","ts":"2026-09-18T10:23:44.139Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.139Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth-token","namespace":"test-ns-xd9fs"}
{"level":"info","ts":"2026-09-18T10:23:44.147Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"21b76109-fae2-4832-8c90-cc193cc87449","rayCluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.147Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"21b76109-fae2-4832-8c90-cc193cc87449","rayCluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.151Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"1dc54e47-17cd-4991-85ec-22344798b40f","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.151Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"a65a2657-b87c-4f67-85e7-e4a9ad79aa05","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.153Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8d1d0ab-f6af-4664-80c0-96bcee7b90ef","name":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:44.153Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8d1d0ab-f6af-4664-80c0-96bcee7b90ef","error":"head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"error","ts":"2026-09-18T10:23:44.153Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8d1d0ab-f6af-4664-80c0-96bcee7b90ef","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.153Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8d1d0ab-f6af-4664-80c0-96bcee7b90ef","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"info","ts":"2026-09-18T10:23:44.153Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8d1d0ab-f6af-4664-80c0-96bcee7b90ef","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.153Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8d1d0ab-f6af-4664-80c0-96bcee7b90ef","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.153Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8d1d0ab-f6af-4664-80c0-96bcee7b90ef","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.153Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8d1d0ab-f6af-4664-80c0-96bcee7b90ef","name":"raycluster-auth-token","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:23:44.155Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"04d6b012-cca2-47ba-a441-a0da4cbea4d2","name":"raycluster-auth-token-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:23:44.159Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"21b76109-fae2-4832-8c90-cc193cc87449","rayCluster":"raycluster-auth-token","secret":"ca-secret"}
{"level":"info","ts":"2026-09-18T10:23:44.159Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"21b76109-fae2-4832-8c90-cc193cc87449","rayCluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.160Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8d1d0ab-f6af-4664-80c0-96bcee7b90ef","name":"raycluster-auth-token","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth-token\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-xd9fs","name":"raycluster-auth-token"}}
{"level":"info","ts":"2026-09-18T10:23:44.160Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8d1d0ab-f6af-4664-80c0-96bcee7b90ef"}
{"level":"error","ts":"2026-09-18T10:23:44.161Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8d1d0ab-f6af-4664-80c0-96bcee7b90ef","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.161Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7abd076f-1352-4808-aa3c-a14b231ff522"}
{"level":"info","ts":"2026-09-18T10:23:44.161Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7abd076f-1352-4808-aa3c-a14b231ff522"}
{"level":"info","ts":"2026-09-18T10:23:44.161Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7abd076f-1352-4808-aa3c-a14b231ff522"}
{"level":"info","ts":"2026-09-18T10:23:44.161Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7abd076f-1352-4808-aa3c-a14b231ff522","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:44.161Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7abd076f-1352-4808-aa3c-a14b231ff522","error":"head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"error","ts":"2026-09-18T10:23:44.161Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7abd076f-1352-4808-aa3c-a14b231ff522","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.161Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7abd076f-1352-4808-aa3c-a14b231ff522","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"info","ts":"2026-09-18T10:23:44.161Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7abd076f-1352-4808-aa3c-a14b231ff522","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.161Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7abd076f-1352-4808-aa3c-a14b231ff522","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.161Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7abd076f-1352-4808-aa3c-a14b231ff522","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.161Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7abd076f-1352-4808-aa3c-a14b231ff522","name":"raycluster-auth-token","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:23:44.166Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"04d6b012-cca2-47ba-a441-a0da4cbea4d2","name":"kuberay-gateway-access","namespace":"test-ns-xd9fs","operation":"created"}
{"level":"info","ts":"2026-09-18T10:23:44.172Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"a65a2657-b87c-4f67-85e7-e4a9ad79aa05","name":"raycluster-auth-token-head"}
{"level":"info","ts":"2026-09-18T10:23:44.173Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"21b76109-fae2-4832-8c90-cc193cc87449","rayCluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.174Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7abd076f-1352-4808-aa3c-a14b231ff522"}
{"level":"error","ts":"2026-09-18T10:23:44.174Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7abd076f-1352-4808-aa3c-a14b231ff522","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.174Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8a7860b-edc3-4cf0-a0ec-e48db04426fd"}
{"level":"info","ts":"2026-09-18T10:23:44.174Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8a7860b-edc3-4cf0-a0ec-e48db04426fd"}
{"level":"info","ts":"2026-09-18T10:23:44.174Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8a7860b-edc3-4cf0-a0ec-e48db04426fd"}
{"level":"info","ts":"2026-09-18T10:23:44.174Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8a7860b-edc3-4cf0-a0ec-e48db04426fd","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:44.174Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8a7860b-edc3-4cf0-a0ec-e48db04426fd","error":"head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"error","ts":"2026-09-18T10:23:44.174Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8a7860b-edc3-4cf0-a0ec-e48db04426fd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.174Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8a7860b-edc3-4cf0-a0ec-e48db04426fd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"info","ts":"2026-09-18T10:23:44.174Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8a7860b-edc3-4cf0-a0ec-e48db04426fd","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.174Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8a7860b-edc3-4cf0-a0ec-e48db04426fd","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.174Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8a7860b-edc3-4cf0-a0ec-e48db04426fd","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.174Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8a7860b-edc3-4cf0-a0ec-e48db04426fd"}
{"level":"error","ts":"2026-09-18T10:23:44.174Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d8a7860b-edc3-4cf0-a0ec-e48db04426fd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.174Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"04d6b012-cca2-47ba-a441-a0da4cbea4d2","name":"test-ns-xd9fs-raycluster-auth-token","operation":"created"}
{"level":"info","ts":"2026-09-18T10:23:44.181Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"04d6b012-cca2-47ba-a441-a0da4cbea4d2","name":"kube-rbac-proxy-config-raycluster-auth-token","operation":"created"}
{"level":"info","ts":"2026-09-18T10:23:44.184Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"e2e20f94-28e1-4f66-9041-8f1b385b9132"}
{"level":"info","ts":"2026-09-18T10:23:44.184Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"e2e20f94-28e1-4f66-9041-8f1b385b9132"}
{"level":"info","ts":"2026-09-18T10:23:44.184Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"e2e20f94-28e1-4f66-9041-8f1b385b9132"}
{"level":"info","ts":"2026-09-18T10:23:44.184Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"e2e20f94-28e1-4f66-9041-8f1b385b9132","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:44.184Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"e2e20f94-28e1-4f66-9041-8f1b385b9132","error":"head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"error","ts":"2026-09-18T10:23:44.184Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"e2e20f94-28e1-4f66-9041-8f1b385b9132","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.184Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"e2e20f94-28e1-4f66-9041-8f1b385b9132","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"info","ts":"2026-09-18T10:23:44.184Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"e2e20f94-28e1-4f66-9041-8f1b385b9132","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.184Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"e2e20f94-28e1-4f66-9041-8f1b385b9132","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.184Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"e2e20f94-28e1-4f66-9041-8f1b385b9132","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.184Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"e2e20f94-28e1-4f66-9041-8f1b385b9132"}
{"level":"error","ts":"2026-09-18T10:23:44.184Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"e2e20f94-28e1-4f66-9041-8f1b385b9132","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.185Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"21b76109-fae2-4832-8c90-cc193cc87449","rayCluster":"raycluster-auth-token","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:23:44.187Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"a65a2657-b87c-4f67-85e7-e4a9ad79aa05","name":"raycluster-auth-token-workers"}
{"level":"info","ts":"2026-09-18T10:23:44.187Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"a65a2657-b87c-4f67-85e7-e4a9ad79aa05","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.187Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"54c9313b-ff9c-4772-8ff1-6b393d9ce335","cluster":"raycluster-auth-token"}
{"level":"error","ts":"2026-09-18T10:23:44.188Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"04d6b012-cca2-47ba-a441-a0da4cbea4d2","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth-token\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.188Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"04d6b012-cca2-47ba-a441-a0da4cbea4d2"}
{"level":"error","ts":"2026-09-18T10:23:44.188Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"04d6b012-cca2-47ba-a441-a0da4cbea4d2","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth-token\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.188Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"cc81673c-3306-45a0-a84a-4bd334a5e267","namespacedName":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"}}
{"level":"info","ts":"2026-09-18T10:23:44.188Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"cc81673c-3306-45a0-a84a-4bd334a5e267","mode":"IntegratedOAuth","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.188Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"cc81673c-3306-45a0-a84a-4bd334a5e267","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.197Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"21b76109-fae2-4832-8c90-cc193cc87449","rayCluster":"raycluster-auth-token","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:23:44.197Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"21b76109-fae2-4832-8c90-cc193cc87449"}
{"level":"info","ts":"2026-09-18T10:23:44.197Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"1d02396f-f2a9-4953-a1dc-ee8dfca63bf1","rayCluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.197Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"1d02396f-f2a9-4953-a1dc-ee8dfca63bf1"}
{"level":"info","ts":"2026-09-18T10:23:44.200Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"cc81673c-3306-45a0-a84a-4bd334a5e267","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.200Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"f0f6fb9f-0181-495a-845c-b30563ffda5e","namespacedName":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"}}
{"level":"info","ts":"2026-09-18T10:23:44.200Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"f0f6fb9f-0181-495a-845c-b30563ffda5e","mode":"IntegratedOAuth","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.200Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"f0f6fb9f-0181-495a-845c-b30563ffda5e","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.201Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"9672fd44-5bc2-432d-8011-5c81b38e8638","rayCluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.201Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"9672fd44-5bc2-432d-8011-5c81b38e8638"}
{"level":"info","ts":"2026-09-18T10:23:44.205Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"54c9313b-ff9c-4772-8ff1-6b393d9ce335","name":"raycluster-auth-token-head"}
{"level":"info","ts":"2026-09-18T10:23:44.266Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"54dd13b2-27b8-4499-a97c-44166a8a19c7"}
{"level":"info","ts":"2026-09-18T10:23:44.266Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"54dd13b2-27b8-4499-a97c-44166a8a19c7"}
{"level":"info","ts":"2026-09-18T10:23:44.266Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"54dd13b2-27b8-4499-a97c-44166a8a19c7"}
{"level":"info","ts":"2026-09-18T10:23:44.266Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"54dd13b2-27b8-4499-a97c-44166a8a19c7","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:44.266Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"54dd13b2-27b8-4499-a97c-44166a8a19c7","error":"head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"error","ts":"2026-09-18T10:23:44.266Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"54dd13b2-27b8-4499-a97c-44166a8a19c7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.266Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"54dd13b2-27b8-4499-a97c-44166a8a19c7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"info","ts":"2026-09-18T10:23:44.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"54dd13b2-27b8-4499-a97c-44166a8a19c7","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"54dd13b2-27b8-4499-a97c-44166a8a19c7","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.266Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"54dd13b2-27b8-4499-a97c-44166a8a19c7","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.266Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"54dd13b2-27b8-4499-a97c-44166a8a19c7"}
{"level":"error","ts":"2026-09-18T10:23:44.266Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"54dd13b2-27b8-4499-a97c-44166a8a19c7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:23:44.266Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"f0f6fb9f-0181-495a-845c-b30563ffda5e","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth-token\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.266Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"f0f6fb9f-0181-495a-845c-b30563ffda5e"}
{"level":"error","ts":"2026-09-18T10:23:44.266Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"f0f6fb9f-0181-495a-845c-b30563ffda5e","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth-token\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.270Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"54c9313b-ff9c-4772-8ff1-6b393d9ce335","name":"raycluster-auth-token-workers"}
{"level":"info","ts":"2026-09-18T10:23:44.270Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"54c9313b-ff9c-4772-8ff1-6b393d9ce335","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.270Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"7f6cd129-b65a-4fc4-b5b2-2504decf918c","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.272Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"b5bfa665-016b-4aae-bf3b-f45b5b0c9165","namespacedName":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"}}
{"level":"info","ts":"2026-09-18T10:23:44.272Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"b5bfa665-016b-4aae-bf3b-f45b5b0c9165","mode":"IntegratedOAuth","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.272Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"b5bfa665-016b-4aae-bf3b-f45b5b0c9165","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.283Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"b5bfa665-016b-4aae-bf3b-f45b5b0c9165","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.284Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"7f6cd129-b65a-4fc4-b5b2-2504decf918c","name":"raycluster-auth-token-head"}
{"level":"info","ts":"2026-09-18T10:23:44.295Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"7f6cd129-b65a-4fc4-b5b2-2504decf918c","name":"raycluster-auth-token-workers"}
{"level":"info","ts":"2026-09-18T10:23:44.295Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"7f6cd129-b65a-4fc4-b5b2-2504decf918c","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:44.346Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"23c5282d-0f40-4372-b912-8e4c60ee17e1"}
{"level":"info","ts":"2026-09-18T10:23:44.346Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"23c5282d-0f40-4372-b912-8e4c60ee17e1"}
{"level":"info","ts":"2026-09-18T10:23:44.346Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"23c5282d-0f40-4372-b912-8e4c60ee17e1"}
{"level":"info","ts":"2026-09-18T10:23:44.346Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"23c5282d-0f40-4372-b912-8e4c60ee17e1","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:44.346Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"23c5282d-0f40-4372-b912-8e4c60ee17e1","error":"head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"error","ts":"2026-09-18T10:23:44.346Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"23c5282d-0f40-4372-b912-8e4c60ee17e1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.346Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"23c5282d-0f40-4372-b912-8e4c60ee17e1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"info","ts":"2026-09-18T10:23:44.346Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"23c5282d-0f40-4372-b912-8e4c60ee17e1","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.346Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"23c5282d-0f40-4372-b912-8e4c60ee17e1","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.346Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"23c5282d-0f40-4372-b912-8e4c60ee17e1","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.346Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"23c5282d-0f40-4372-b912-8e4c60ee17e1"}
{"level":"error","ts":"2026-09-18T10:23:44.346Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"23c5282d-0f40-4372-b912-8e4c60ee17e1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.507Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"a32a3352-07f4-41dd-9f83-fa7abc2cc32d"}
{"level":"info","ts":"2026-09-18T10:23:44.507Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"a32a3352-07f4-41dd-9f83-fa7abc2cc32d"}
{"level":"info","ts":"2026-09-18T10:23:44.507Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"a32a3352-07f4-41dd-9f83-fa7abc2cc32d"}
{"level":"info","ts":"2026-09-18T10:23:44.507Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"a32a3352-07f4-41dd-9f83-fa7abc2cc32d","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:44.508Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"a32a3352-07f4-41dd-9f83-fa7abc2cc32d","error":"head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"error","ts":"2026-09-18T10:23:44.508Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"a32a3352-07f4-41dd-9f83-fa7abc2cc32d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.508Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"a32a3352-07f4-41dd-9f83-fa7abc2cc32d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"info","ts":"2026-09-18T10:23:44.508Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"a32a3352-07f4-41dd-9f83-fa7abc2cc32d","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.508Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"a32a3352-07f4-41dd-9f83-fa7abc2cc32d","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.508Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"a32a3352-07f4-41dd-9f83-fa7abc2cc32d","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.508Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"a32a3352-07f4-41dd-9f83-fa7abc2cc32d"}
{"level":"error","ts":"2026-09-18T10:23:44.508Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"a32a3352-07f4-41dd-9f83-fa7abc2cc32d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.828Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"2f54c436-1ee3-4849-b997-54ff9c91b3e5"}
{"level":"info","ts":"2026-09-18T10:23:44.828Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"2f54c436-1ee3-4849-b997-54ff9c91b3e5"}
{"level":"info","ts":"2026-09-18T10:23:44.829Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"2f54c436-1ee3-4849-b997-54ff9c91b3e5"}
{"level":"info","ts":"2026-09-18T10:23:44.829Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"2f54c436-1ee3-4849-b997-54ff9c91b3e5","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:44.829Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"2f54c436-1ee3-4849-b997-54ff9c91b3e5","error":"head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"error","ts":"2026-09-18T10:23:44.829Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"2f54c436-1ee3-4849-b997-54ff9c91b3e5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:44.829Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"2f54c436-1ee3-4849-b997-54ff9c91b3e5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"info","ts":"2026-09-18T10:23:44.829Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"2f54c436-1ee3-4849-b997-54ff9c91b3e5","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.829Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"2f54c436-1ee3-4849-b997-54ff9c91b3e5","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.829Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"2f54c436-1ee3-4849-b997-54ff9c91b3e5","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:44.829Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"2f54c436-1ee3-4849-b997-54ff9c91b3e5"}
{"level":"error","ts":"2026-09-18T10:23:44.829Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"2f54c436-1ee3-4849-b997-54ff9c91b3e5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:45.470Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"aeda1bac-4858-4710-ba25-3bc3235ee818"}
{"level":"info","ts":"2026-09-18T10:23:45.470Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"aeda1bac-4858-4710-ba25-3bc3235ee818"}
{"level":"info","ts":"2026-09-18T10:23:45.470Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"aeda1bac-4858-4710-ba25-3bc3235ee818"}
{"level":"info","ts":"2026-09-18T10:23:45.470Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"aeda1bac-4858-4710-ba25-3bc3235ee818","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:45.470Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"aeda1bac-4858-4710-ba25-3bc3235ee818","error":"head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"error","ts":"2026-09-18T10:23:45.470Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"aeda1bac-4858-4710-ba25-3bc3235ee818","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:45.470Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"aeda1bac-4858-4710-ba25-3bc3235ee818","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found"}
{"level":"info","ts":"2026-09-18T10:23:45.470Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"aeda1bac-4858-4710-ba25-3bc3235ee818","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:45.470Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"aeda1bac-4858-4710-ba25-3bc3235ee818","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:45.470Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"aeda1bac-4858-4710-ba25-3bc3235ee818","filter labels":["test-ns-xd9fs",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:23:45.470Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"aeda1bac-4858-4710-ba25-3bc3235ee818"}
{"level":"error","ts":"2026-09-18T10:23:45.470Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"aeda1bac-4858-4710-ba25-3bc3235ee818","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth-token not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:46.751Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad"}
{"level":"info","ts":"2026-09-18T10:23:46.751Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad"}
{"level":"info","ts":"2026-09-18T10:23:46.751Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad"}
{"level":"info","ts":"2026-09-18T10:23:46.751Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:46.752Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad"}
{"level":"info","ts":"2026-09-18T10:23:46.752Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:23:46.752Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad"}
{"level":"info","ts":"2026-09-18T10:23:46.752Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","mode":"IntegratedOAuth","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:46.752Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","cluster":"raycluster-auth-token","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:23:46.752Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","cluster":"raycluster-auth-token","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-token-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-09-18T10:23:46.752Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth-token","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-token-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:23:46.752Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-09-18T10:23:46.752Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:23:46.752Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-09-18T10:23:46.752Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","enabled":true}
{"level":"info","ts":"2026-09-18T10:23:46.774Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","name":"raycluster-auth-token-head-xbzkj"}
{"level":"info","ts":"2026-09-18T10:23:46.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:23:46.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:46.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-09-18T10:23:46.774Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:23:46.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:46.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-09-18T10:23:46.774Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad"}
{"level":"info","ts":"2026-09-18T10:23:46.774Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-token-head-svc.test-ns-xd9fs.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-09-18T10:23:46.774Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","rayStartCmd":"ray start  --address=raycluster-auth-token-head-svc.test-ns-xd9fs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:23:46.774Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-token-head-svc.test-ns-xd9fs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-09-18T10:23:46.774Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","enabled":true}
{"level":"info","ts":"2026-09-18T10:23:46.789Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","name":"raycluster-auth-token-small-group-worker-2k4hv"}
{"level":"info","ts":"2026-09-18T10:23:46.789Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:46.789Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"13fdf3fe-c427-4f29-8871-fc27f27756ad","name":"raycluster-auth-token","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:23:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-token-head-xbzkj","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:23:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:23:46.802Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2"}
{"level":"info","ts":"2026-09-18T10:23:46.802Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2"}
{"level":"info","ts":"2026-09-18T10:23:46.802Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2"}
{"level":"info","ts":"2026-09-18T10:23:46.802Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:46.802Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:46.802Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:46.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:23:46.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:46.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:46.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:46.803Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:23:46.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:46.803Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"2d72e51d-6c82-4fda-bacc-e822002020df","rayCluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:46.803Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:46.803Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:46.803Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"2d72e51d-6c82-4fda-bacc-e822002020df","rayCluster":"raycluster-auth-token","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:23:46.803Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2","name":"raycluster-auth-token","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:23:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-token-head-xbzkj","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:23:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:23:46.803Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"5c07f116-7a23-4f3a-9221-68847a95cd17","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:46.866Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2","name":"raycluster-auth-token","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth-token\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-xd9fs","name":"raycluster-auth-token"}}
{"level":"info","ts":"2026-09-18T10:23:46.866Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2"}
{"level":"error","ts":"2026-09-18T10:23:46.866Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ec4bb169-d5c8-4d1f-b804-1392c93696d2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth-token\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:23:46.866Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"382facbf-9d95-4a6c-a637-687d832afd36"}
{"level":"info","ts":"2026-09-18T10:23:46.866Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"382facbf-9d95-4a6c-a637-687d832afd36"}
{"level":"info","ts":"2026-09-18T10:23:46.866Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"382facbf-9d95-4a6c-a637-687d832afd36"}
{"level":"info","ts":"2026-09-18T10:23:46.866Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"382facbf-9d95-4a6c-a637-687d832afd36","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:46.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"382facbf-9d95-4a6c-a637-687d832afd36","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:46.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"382facbf-9d95-4a6c-a637-687d832afd36","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:46.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"382facbf-9d95-4a6c-a637-687d832afd36","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:23:46.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"382facbf-9d95-4a6c-a637-687d832afd36","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:46.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"382facbf-9d95-4a6c-a637-687d832afd36","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:46.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"382facbf-9d95-4a6c-a637-687d832afd36","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:46.867Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"382facbf-9d95-4a6c-a637-687d832afd36","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:23:46.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"382facbf-9d95-4a6c-a637-687d832afd36","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:46.867Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"382facbf-9d95-4a6c-a637-687d832afd36","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:46.867Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"382facbf-9d95-4a6c-a637-687d832afd36","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:46.867Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"382facbf-9d95-4a6c-a637-687d832afd36","name":"raycluster-auth-token","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-token-head-xbzkj","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:23:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-token-head-xbzkj","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:23:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:23:46.871Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"5c07f116-7a23-4f3a-9221-68847a95cd17","name":"raycluster-auth-token-head"}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"20d9e126-3aaf-435b-8e17-4437ea7377c7"}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"20d9e126-3aaf-435b-8e17-4437ea7377c7"}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"20d9e126-3aaf-435b-8e17-4437ea7377c7"}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"20d9e126-3aaf-435b-8e17-4437ea7377c7","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"20d9e126-3aaf-435b-8e17-4437ea7377c7","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"20d9e126-3aaf-435b-8e17-4437ea7377c7","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"20d9e126-3aaf-435b-8e17-4437ea7377c7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"20d9e126-3aaf-435b-8e17-4437ea7377c7","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"20d9e126-3aaf-435b-8e17-4437ea7377c7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"20d9e126-3aaf-435b-8e17-4437ea7377c7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"20d9e126-3aaf-435b-8e17-4437ea7377c7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"20d9e126-3aaf-435b-8e17-4437ea7377c7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"20d9e126-3aaf-435b-8e17-4437ea7377c7","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"20d9e126-3aaf-435b-8e17-4437ea7377c7","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"20d9e126-3aaf-435b-8e17-4437ea7377c7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"20d9e126-3aaf-435b-8e17-4437ea7377c7","seconds":300}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"b145931f-af4d-4403-bd83-07784be1de27","rayCluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:46.969Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"b145931f-af4d-4403-bd83-07784be1de27","rayCluster":"raycluster-auth-token","podIPs":[]}
{"level":"info","ts":"2026-09-18T10:23:46.972Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"5c07f116-7a23-4f3a-9221-68847a95cd17","name":"raycluster-auth-token-workers"}
{"level":"info","ts":"2026-09-18T10:23:46.972Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"5c07f116-7a23-4f3a-9221-68847a95cd17","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:46.972Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"6908e0d7-3229-44bb-9ace-39357f0d1be1","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:46.982Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"6908e0d7-3229-44bb-9ace-39357f0d1be1","name":"raycluster-auth-token-head"}
{"level":"info","ts":"2026-09-18T10:23:47.070Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"6908e0d7-3229-44bb-9ace-39357f0d1be1","name":"raycluster-auth-token-workers"}
{"level":"info","ts":"2026-09-18T10:23:47.071Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"6908e0d7-3229-44bb-9ace-39357f0d1be1","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:47.309Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e516053e-6ee0-4148-bc1d-419ebd9e561d"}
{"level":"info","ts":"2026-09-18T10:23:47.309Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e516053e-6ee0-4148-bc1d-419ebd9e561d"}
{"level":"info","ts":"2026-09-18T10:23:47.309Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e516053e-6ee0-4148-bc1d-419ebd9e561d"}
{"level":"info","ts":"2026-09-18T10:23:47.309Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e516053e-6ee0-4148-bc1d-419ebd9e561d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:47.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e516053e-6ee0-4148-bc1d-419ebd9e561d","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:47.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e516053e-6ee0-4148-bc1d-419ebd9e561d","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:47.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e516053e-6ee0-4148-bc1d-419ebd9e561d","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:23:47.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e516053e-6ee0-4148-bc1d-419ebd9e561d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:47.309Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e516053e-6ee0-4148-bc1d-419ebd9e561d","namespace":"test-ns-wlgxb","name":"ray-cluster-small-group-worker-9c9dl"}
{"level":"info","ts":"2026-09-18T10:23:47.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e516053e-6ee0-4148-bc1d-419ebd9e561d","The worker Pod has already been deleted":"ray-cluster-small-group-worker-9c9dl"}
{"level":"info","ts":"2026-09-18T10:23:47.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e516053e-6ee0-4148-bc1d-419ebd9e561d","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:47.312Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e516053e-6ee0-4148-bc1d-419ebd9e561d","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:23:47.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e516053e-6ee0-4148-bc1d-419ebd9e561d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:47.313Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e516053e-6ee0-4148-bc1d-419ebd9e561d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:23:47.313Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"e516053e-6ee0-4148-bc1d-419ebd9e561d","seconds":300}
{"level":"info","ts":"2026-09-18T10:23:47.343Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bc87dce1-a8a4-46e5-b5eb-58f096e8bf5a"}
{"level":"info","ts":"2026-09-18T10:23:47.343Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bc87dce1-a8a4-46e5-b5eb-58f096e8bf5a"}
{"level":"info","ts":"2026-09-18T10:23:47.343Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bc87dce1-a8a4-46e5-b5eb-58f096e8bf5a"}
{"level":"info","ts":"2026-09-18T10:23:47.343Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bc87dce1-a8a4-46e5-b5eb-58f096e8bf5a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:47.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bc87dce1-a8a4-46e5-b5eb-58f096e8bf5a","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:47.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bc87dce1-a8a4-46e5-b5eb-58f096e8bf5a","head Pod":"ray-cluster-head-4f7tw","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-4f7tw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:47.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bc87dce1-a8a4-46e5-b5eb-58f096e8bf5a","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:23:47.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bc87dce1-a8a4-46e5-b5eb-58f096e8bf5a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:47.343Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bc87dce1-a8a4-46e5-b5eb-58f096e8bf5a","namespace":"test-ns-wlgxb","name":"ray-cluster-small-group-worker-9c9dl"}
{"level":"info","ts":"2026-09-18T10:23:47.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bc87dce1-a8a4-46e5-b5eb-58f096e8bf5a","The worker Pod has already been deleted":"ray-cluster-small-group-worker-9c9dl"}
{"level":"info","ts":"2026-09-18T10:23:47.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bc87dce1-a8a4-46e5-b5eb-58f096e8bf5a","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:47.347Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bc87dce1-a8a4-46e5-b5eb-58f096e8bf5a","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:23:47.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bc87dce1-a8a4-46e5-b5eb-58f096e8bf5a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:47.347Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bc87dce1-a8a4-46e5-b5eb-58f096e8bf5a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:23:47.347Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bc87dce1-a8a4-46e5-b5eb-58f096e8bf5a","seconds":300}
{"level":"info","ts":"2026-09-18T10:23:47.379Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"c0d34971-4458-4a57-ba2b-f2a6512c4812"}
{"level":"info","ts":"2026-09-18T10:23:47.379Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"c0d34971-4458-4a57-ba2b-f2a6512c4812"}
{"level":"info","ts":"2026-09-18T10:23:47.380Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"c0d34971-4458-4a57-ba2b-f2a6512c4812"}
{"level":"info","ts":"2026-09-18T10:23:47.380Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"c0d34971-4458-4a57-ba2b-f2a6512c4812","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:47.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"c0d34971-4458-4a57-ba2b-f2a6512c4812","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:47.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"c0d34971-4458-4a57-ba2b-f2a6512c4812","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:47.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"c0d34971-4458-4a57-ba2b-f2a6512c4812","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:23:47.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"c0d34971-4458-4a57-ba2b-f2a6512c4812","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:47.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"c0d34971-4458-4a57-ba2b-f2a6512c4812","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:47.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"c0d34971-4458-4a57-ba2b-f2a6512c4812","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:47.380Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"c0d34971-4458-4a57-ba2b-f2a6512c4812","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:23:47.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"c0d34971-4458-4a57-ba2b-f2a6512c4812","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:47.380Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"c0d34971-4458-4a57-ba2b-f2a6512c4812","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:47.380Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"c0d34971-4458-4a57-ba2b-f2a6512c4812","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:47.380Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"c0d34971-4458-4a57-ba2b-f2a6512c4812","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:23:47.380Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"c0d34971-4458-4a57-ba2b-f2a6512c4812","seconds":300}
{"level":"info","ts":"2026-09-18T10:23:47.385Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f5924398-2f3c-4e84-b4a3-1eb8778567e1"}
{"level":"info","ts":"2026-09-18T10:23:47.385Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f5924398-2f3c-4e84-b4a3-1eb8778567e1"}
{"level":"info","ts":"2026-09-18T10:23:47.385Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f5924398-2f3c-4e84-b4a3-1eb8778567e1"}
{"level":"info","ts":"2026-09-18T10:23:47.385Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f5924398-2f3c-4e84-b4a3-1eb8778567e1","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:47.386Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f5924398-2f3c-4e84-b4a3-1eb8778567e1","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:47.386Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f5924398-2f3c-4e84-b4a3-1eb8778567e1","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:47.386Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f5924398-2f3c-4e84-b4a3-1eb8778567e1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:23:47.386Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f5924398-2f3c-4e84-b4a3-1eb8778567e1","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:47.386Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f5924398-2f3c-4e84-b4a3-1eb8778567e1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:47.386Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f5924398-2f3c-4e84-b4a3-1eb8778567e1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:47.386Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f5924398-2f3c-4e84-b4a3-1eb8778567e1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:23:47.386Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f5924398-2f3c-4e84-b4a3-1eb8778567e1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:47.386Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f5924398-2f3c-4e84-b4a3-1eb8778567e1","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:47.386Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f5924398-2f3c-4e84-b4a3-1eb8778567e1","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:47.386Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f5924398-2f3c-4e84-b4a3-1eb8778567e1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:23:47.386Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f5924398-2f3c-4e84-b4a3-1eb8778567e1","seconds":300}
{"level":"info","ts":"2026-09-18T10:23:47.851Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"30fed1f1-48eb-4c9d-a7a8-e1e88de985ea"}
{"level":"info","ts":"2026-09-18T10:23:47.851Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"30fed1f1-48eb-4c9d-a7a8-e1e88de985ea"}
{"level":"info","ts":"2026-09-18T10:23:47.851Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"30fed1f1-48eb-4c9d-a7a8-e1e88de985ea"}
{"level":"info","ts":"2026-09-18T10:23:47.851Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"30fed1f1-48eb-4c9d-a7a8-e1e88de985ea","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:47.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"30fed1f1-48eb-4c9d-a7a8-e1e88de985ea","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:47.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"30fed1f1-48eb-4c9d-a7a8-e1e88de985ea","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:47.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"30fed1f1-48eb-4c9d-a7a8-e1e88de985ea","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:23:47.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"30fed1f1-48eb-4c9d-a7a8-e1e88de985ea","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:47.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"30fed1f1-48eb-4c9d-a7a8-e1e88de985ea","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:47.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"30fed1f1-48eb-4c9d-a7a8-e1e88de985ea","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:47.852Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"30fed1f1-48eb-4c9d-a7a8-e1e88de985ea","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:23:47.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"30fed1f1-48eb-4c9d-a7a8-e1e88de985ea","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:47.852Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"30fed1f1-48eb-4c9d-a7a8-e1e88de985ea","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:47.852Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"30fed1f1-48eb-4c9d-a7a8-e1e88de985ea","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:47.852Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"30fed1f1-48eb-4c9d-a7a8-e1e88de985ea","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:23:47.852Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"30fed1f1-48eb-4c9d-a7a8-e1e88de985ea","seconds":300}
{"level":"info","ts":"2026-09-18T10:23:48.058Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"de69fc33-2e48-444b-a00a-faf92c41ddb8"}
{"level":"info","ts":"2026-09-18T10:23:48.058Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"de69fc33-2e48-444b-a00a-faf92c41ddb8"}
{"level":"info","ts":"2026-09-18T10:23:48.058Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"de69fc33-2e48-444b-a00a-faf92c41ddb8"}
{"level":"info","ts":"2026-09-18T10:23:48.058Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"de69fc33-2e48-444b-a00a-faf92c41ddb8","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:48.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"de69fc33-2e48-444b-a00a-faf92c41ddb8","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:48.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"de69fc33-2e48-444b-a00a-faf92c41ddb8","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:48.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"de69fc33-2e48-444b-a00a-faf92c41ddb8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:23:48.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"de69fc33-2e48-444b-a00a-faf92c41ddb8","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:48.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"de69fc33-2e48-444b-a00a-faf92c41ddb8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:48.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"de69fc33-2e48-444b-a00a-faf92c41ddb8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:48.058Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"de69fc33-2e48-444b-a00a-faf92c41ddb8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:23:48.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"de69fc33-2e48-444b-a00a-faf92c41ddb8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:48.058Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"de69fc33-2e48-444b-a00a-faf92c41ddb8","podName":"raycluster-auth-token-small-group-worker-2k4hv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:48.058Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"de69fc33-2e48-444b-a00a-faf92c41ddb8","podName":"raycluster-auth-token-small-group-worker-2k4hv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:48.058Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"de69fc33-2e48-444b-a00a-faf92c41ddb8","name":"raycluster-auth-token","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-token-head-xbzkj","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:23:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.62","serviceIP":"10.132.0.62","podName":"raycluster-auth-token-head-xbzkj","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:23:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:23:48.078Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"aea61fc0-a287-4701-9eae-3a48c650f9cf","rayCluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:48.078Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"abcbe66d-f6f0-4428-92f8-3d1ec264f7d0","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:48.079Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"aea61fc0-a287-4701-9eae-3a48c650f9cf","pod":"raycluster-auth-token-head-xbzkj","ip":"10.132.0.62"}
{"level":"info","ts":"2026-09-18T10:23:48.079Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"aea61fc0-a287-4701-9eae-3a48c650f9cf","pod":"raycluster-auth-token-small-group-worker-2k4hv","ip":"10.133.0.54"}
{"level":"info","ts":"2026-09-18T10:23:48.088Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"aea61fc0-a287-4701-9eae-3a48c650f9cf","certificate":"ray-head-cert-raycluster-auth-token","dnsNames":["raycluster-auth-token-head-svc","localhost","raycluster-auth-token-head-svc.test-ns-xd9fs.svc","raycluster-auth-token-head-svc.test-ns-xd9fs.svc.cluster.local"],"ipAddresses":["10.132.0.62","10.133.0.54","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:23:48.092Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"abcbe66d-f6f0-4428-92f8-3d1ec264f7d0","name":"raycluster-auth-token-head"}
{"level":"info","ts":"2026-09-18T10:23:48.102Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"aea61fc0-a287-4701-9eae-3a48c650f9cf","certificate":"ray-worker-cert-raycluster-auth-token","dnsNames":["raycluster-auth-token-worker-svc","localhost","raycluster-auth-token-small-group","raycluster-auth-token-worker-svc.test-ns-xd9fs.svc","raycluster-auth-token-worker-svc.test-ns-xd9fs.svc.cluster.local","*.raycluster-auth-token-worker-svc.test-ns-xd9fs.svc","*.raycluster-auth-token-worker-svc.test-ns-xd9fs.svc.cluster.local","*-worker-*.test-ns-xd9fs.svc","*-worker-*.test-ns-xd9fs.svc.cluster.local"],"ipAddresses":["10.132.0.62","10.133.0.54","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:23:48.102Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"aea61fc0-a287-4701-9eae-3a48c650f9cf","rayCluster":"raycluster-auth-token","podIPs":["10.132.0.62","10.133.0.54"]}
{"level":"info","ts":"2026-09-18T10:23:48.105Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"abcbe66d-f6f0-4428-92f8-3d1ec264f7d0","name":"raycluster-auth-token-workers"}
{"level":"info","ts":"2026-09-18T10:23:48.105Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"abcbe66d-f6f0-4428-92f8-3d1ec264f7d0","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:48.978Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f99ccc45-b4a5-4657-97cb-d3f4373fe8d6"}
{"level":"info","ts":"2026-09-18T10:23:48.978Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f99ccc45-b4a5-4657-97cb-d3f4373fe8d6"}
{"level":"info","ts":"2026-09-18T10:23:48.978Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f99ccc45-b4a5-4657-97cb-d3f4373fe8d6"}
{"level":"info","ts":"2026-09-18T10:23:48.978Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f99ccc45-b4a5-4657-97cb-d3f4373fe8d6","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:48.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f99ccc45-b4a5-4657-97cb-d3f4373fe8d6","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:48.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f99ccc45-b4a5-4657-97cb-d3f4373fe8d6","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:48.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f99ccc45-b4a5-4657-97cb-d3f4373fe8d6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:23:48.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f99ccc45-b4a5-4657-97cb-d3f4373fe8d6","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:48.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f99ccc45-b4a5-4657-97cb-d3f4373fe8d6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:48.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f99ccc45-b4a5-4657-97cb-d3f4373fe8d6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:48.978Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f99ccc45-b4a5-4657-97cb-d3f4373fe8d6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:23:48.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f99ccc45-b4a5-4657-97cb-d3f4373fe8d6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:48.978Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f99ccc45-b4a5-4657-97cb-d3f4373fe8d6","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:48.978Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f99ccc45-b4a5-4657-97cb-d3f4373fe8d6","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:48.978Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f99ccc45-b4a5-4657-97cb-d3f4373fe8d6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:23:48.978Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f99ccc45-b4a5-4657-97cb-d3f4373fe8d6","seconds":300}
{"level":"info","ts":"2026-09-18T10:23:52.879Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f936d5cd-6397-4ec8-9134-64df246e00c9"}
{"level":"info","ts":"2026-09-18T10:23:52.879Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f936d5cd-6397-4ec8-9134-64df246e00c9"}
{"level":"info","ts":"2026-09-18T10:23:52.879Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f936d5cd-6397-4ec8-9134-64df246e00c9"}
{"level":"info","ts":"2026-09-18T10:23:52.879Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f936d5cd-6397-4ec8-9134-64df246e00c9","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:52.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f936d5cd-6397-4ec8-9134-64df246e00c9","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:52.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f936d5cd-6397-4ec8-9134-64df246e00c9","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:52.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f936d5cd-6397-4ec8-9134-64df246e00c9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:23:52.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f936d5cd-6397-4ec8-9134-64df246e00c9","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:52.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f936d5cd-6397-4ec8-9134-64df246e00c9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:52.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f936d5cd-6397-4ec8-9134-64df246e00c9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:52.880Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f936d5cd-6397-4ec8-9134-64df246e00c9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:23:52.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f936d5cd-6397-4ec8-9134-64df246e00c9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:52.880Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f936d5cd-6397-4ec8-9134-64df246e00c9","podName":"raycluster-auth-token-small-group-worker-2k4hv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:52.880Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f936d5cd-6397-4ec8-9134-64df246e00c9","podName":"raycluster-auth-token-small-group-worker-2k4hv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:52.880Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f936d5cd-6397-4ec8-9134-64df246e00c9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:23:52.880Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"f936d5cd-6397-4ec8-9134-64df246e00c9","seconds":300}
{"level":"info","ts":"2026-09-18T10:23:53.084Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"6fc659d8-bf55-4ead-a186-233ead67dbe1"}
{"level":"info","ts":"2026-09-18T10:23:53.084Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"6fc659d8-bf55-4ead-a186-233ead67dbe1"}
{"level":"info","ts":"2026-09-18T10:23:53.084Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"6fc659d8-bf55-4ead-a186-233ead67dbe1"}
{"level":"info","ts":"2026-09-18T10:23:53.084Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"6fc659d8-bf55-4ead-a186-233ead67dbe1","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:53.084Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"6fc659d8-bf55-4ead-a186-233ead67dbe1","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:53.084Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"6fc659d8-bf55-4ead-a186-233ead67dbe1","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:53.084Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"6fc659d8-bf55-4ead-a186-233ead67dbe1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:23:53.084Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"6fc659d8-bf55-4ead-a186-233ead67dbe1","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:53.084Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"6fc659d8-bf55-4ead-a186-233ead67dbe1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:53.084Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"6fc659d8-bf55-4ead-a186-233ead67dbe1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:53.084Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"6fc659d8-bf55-4ead-a186-233ead67dbe1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:23:53.084Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"6fc659d8-bf55-4ead-a186-233ead67dbe1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:53.085Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"6fc659d8-bf55-4ead-a186-233ead67dbe1","podName":"raycluster-auth-token-small-group-worker-2k4hv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:53.085Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"6fc659d8-bf55-4ead-a186-233ead67dbe1","podName":"raycluster-auth-token-small-group-worker-2k4hv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:53.085Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"6fc659d8-bf55-4ead-a186-233ead67dbe1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:23:53.085Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"6fc659d8-bf55-4ead-a186-233ead67dbe1","seconds":300}
{"level":"info","ts":"2026-09-18T10:23:53.895Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"3b9189a8-c642-4581-b70a-941753eb7d86"}
{"level":"info","ts":"2026-09-18T10:23:53.895Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"3b9189a8-c642-4581-b70a-941753eb7d86"}
{"level":"info","ts":"2026-09-18T10:23:53.895Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"3b9189a8-c642-4581-b70a-941753eb7d86"}
{"level":"info","ts":"2026-09-18T10:23:53.895Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"3b9189a8-c642-4581-b70a-941753eb7d86","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:53.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"3b9189a8-c642-4581-b70a-941753eb7d86","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:53.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"3b9189a8-c642-4581-b70a-941753eb7d86","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:53.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"3b9189a8-c642-4581-b70a-941753eb7d86","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:23:53.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"3b9189a8-c642-4581-b70a-941753eb7d86","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:53.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"3b9189a8-c642-4581-b70a-941753eb7d86","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:53.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"3b9189a8-c642-4581-b70a-941753eb7d86","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:53.895Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"3b9189a8-c642-4581-b70a-941753eb7d86","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:23:53.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"3b9189a8-c642-4581-b70a-941753eb7d86","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:53.896Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"3b9189a8-c642-4581-b70a-941753eb7d86","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:53.896Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"3b9189a8-c642-4581-b70a-941753eb7d86","podName":"raycluster-auth-token-head-xbzkj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-09-18T10:23:53.896Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"3b9189a8-c642-4581-b70a-941753eb7d86","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:23:53.896Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"3b9189a8-c642-4581-b70a-941753eb7d86","seconds":300}
{"level":"info","ts":"2026-09-18T10:23:54.089Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"398fc412-b3da-478e-9f93-23309a62f82d"}
{"level":"info","ts":"2026-09-18T10:23:54.089Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"398fc412-b3da-478e-9f93-23309a62f82d"}
{"level":"info","ts":"2026-09-18T10:23:54.089Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"398fc412-b3da-478e-9f93-23309a62f82d"}
{"level":"info","ts":"2026-09-18T10:23:54.089Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"398fc412-b3da-478e-9f93-23309a62f82d","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:54.089Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"398fc412-b3da-478e-9f93-23309a62f82d","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:54.089Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"398fc412-b3da-478e-9f93-23309a62f82d","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:54.089Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"398fc412-b3da-478e-9f93-23309a62f82d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:23:54.089Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"398fc412-b3da-478e-9f93-23309a62f82d","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:54.089Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"398fc412-b3da-478e-9f93-23309a62f82d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:54.089Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"398fc412-b3da-478e-9f93-23309a62f82d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:54.089Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"398fc412-b3da-478e-9f93-23309a62f82d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:23:54.089Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"398fc412-b3da-478e-9f93-23309a62f82d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:54.089Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"398fc412-b3da-478e-9f93-23309a62f82d","podName":"raycluster-auth-token-head-xbzkj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:23:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:23:54.089Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"398fc412-b3da-478e-9f93-23309a62f82d","podName":"raycluster-auth-token-head-xbzkj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:23:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:23:54.089Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"398fc412-b3da-478e-9f93-23309a62f82d","name":"raycluster-auth-token","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.62","serviceIP":"10.132.0.62","podName":"raycluster-auth-token-head-xbzkj","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:23:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.62","serviceIP":"10.132.0.62","podName":"raycluster-auth-token-head-xbzkj","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:23:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:23:54.102Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"71484de9-a061-4ab9-8498-4fe519968c62","rayCluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:54.102Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"fbafe210-71cd-40d7-b51a-34eb2dbb585a","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:54.102Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"71484de9-a061-4ab9-8498-4fe519968c62","pod":"raycluster-auth-token-head-xbzkj","ip":"10.132.0.62"}
{"level":"info","ts":"2026-09-18T10:23:54.102Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"71484de9-a061-4ab9-8498-4fe519968c62","pod":"raycluster-auth-token-small-group-worker-2k4hv","ip":"10.133.0.54"}
{"level":"info","ts":"2026-09-18T10:23:54.102Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"71484de9-a061-4ab9-8498-4fe519968c62","rayCluster":"raycluster-auth-token","podIPs":["10.132.0.62","10.133.0.54"]}
{"level":"info","ts":"2026-09-18T10:23:54.114Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"fbafe210-71cd-40d7-b51a-34eb2dbb585a","name":"raycluster-auth-token-head"}
{"level":"info","ts":"2026-09-18T10:23:54.126Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"fbafe210-71cd-40d7-b51a-34eb2dbb585a","name":"raycluster-auth-token-workers"}
{"level":"info","ts":"2026-09-18T10:23:54.126Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"fbafe210-71cd-40d7-b51a-34eb2dbb585a","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:23:56.102Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ba6f2305-c768-4658-b0ee-d0153a207dc4"}
{"level":"info","ts":"2026-09-18T10:23:56.102Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ba6f2305-c768-4658-b0ee-d0153a207dc4"}
{"level":"info","ts":"2026-09-18T10:23:56.102Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ba6f2305-c768-4658-b0ee-d0153a207dc4"}
{"level":"info","ts":"2026-09-18T10:23:56.102Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ba6f2305-c768-4658-b0ee-d0153a207dc4","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:23:56.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ba6f2305-c768-4658-b0ee-d0153a207dc4","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:23:56.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ba6f2305-c768-4658-b0ee-d0153a207dc4","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:56.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ba6f2305-c768-4658-b0ee-d0153a207dc4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:23:56.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ba6f2305-c768-4658-b0ee-d0153a207dc4","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:23:56.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ba6f2305-c768-4658-b0ee-d0153a207dc4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:56.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ba6f2305-c768-4658-b0ee-d0153a207dc4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:23:56.103Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ba6f2305-c768-4658-b0ee-d0153a207dc4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:23:56.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ba6f2305-c768-4658-b0ee-d0153a207dc4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:23:56.103Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ba6f2305-c768-4658-b0ee-d0153a207dc4","podName":"raycluster-auth-token-head-xbzkj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:23:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:23:56.103Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ba6f2305-c768-4658-b0ee-d0153a207dc4","podName":"raycluster-auth-token-head-xbzkj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:23:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:23:56.103Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ba6f2305-c768-4658-b0ee-d0153a207dc4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:23:56.103Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"ba6f2305-c768-4658-b0ee-d0153a207dc4","seconds":300}
{"level":"info","ts":"2026-09-18T10:24:02.933Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d875a171-38eb-4889-a6ee-b63fb60a789c"}
{"level":"info","ts":"2026-09-18T10:24:02.933Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d875a171-38eb-4889-a6ee-b63fb60a789c"}
{"level":"info","ts":"2026-09-18T10:24:02.933Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d875a171-38eb-4889-a6ee-b63fb60a789c"}
{"level":"info","ts":"2026-09-18T10:24:02.933Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d875a171-38eb-4889-a6ee-b63fb60a789c","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:02.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d875a171-38eb-4889-a6ee-b63fb60a789c","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:24:02.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d875a171-38eb-4889-a6ee-b63fb60a789c","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:24:02.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d875a171-38eb-4889-a6ee-b63fb60a789c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:24:02.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d875a171-38eb-4889-a6ee-b63fb60a789c","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:24:02.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d875a171-38eb-4889-a6ee-b63fb60a789c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:24:02.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d875a171-38eb-4889-a6ee-b63fb60a789c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:24:02.933Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d875a171-38eb-4889-a6ee-b63fb60a789c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:24:02.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d875a171-38eb-4889-a6ee-b63fb60a789c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:24:02.933Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d875a171-38eb-4889-a6ee-b63fb60a789c","podName":"raycluster-auth-token-head-xbzkj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:23:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:24:02.934Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d875a171-38eb-4889-a6ee-b63fb60a789c","podName":"raycluster-auth-token-head-xbzkj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:23:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-09-18T10:24:02.934Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d875a171-38eb-4889-a6ee-b63fb60a789c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:24:02.934Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"d875a171-38eb-4889-a6ee-b63fb60a789c","seconds":300}
{"level":"info","ts":"2026-09-18T10:24:03.938Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"388a176c-ca0b-493c-bded-1351d4f5bd27"}
{"level":"info","ts":"2026-09-18T10:24:03.938Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"388a176c-ca0b-493c-bded-1351d4f5bd27"}
{"level":"info","ts":"2026-09-18T10:24:03.938Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"388a176c-ca0b-493c-bded-1351d4f5bd27"}
{"level":"info","ts":"2026-09-18T10:24:03.938Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"388a176c-ca0b-493c-bded-1351d4f5bd27","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:03.938Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"388a176c-ca0b-493c-bded-1351d4f5bd27","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:24:03.938Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"388a176c-ca0b-493c-bded-1351d4f5bd27","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:24:03.938Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"388a176c-ca0b-493c-bded-1351d4f5bd27","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:24:03.938Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"388a176c-ca0b-493c-bded-1351d4f5bd27","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:24:03.939Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"388a176c-ca0b-493c-bded-1351d4f5bd27","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:24:03.939Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"388a176c-ca0b-493c-bded-1351d4f5bd27","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:24:03.939Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"388a176c-ca0b-493c-bded-1351d4f5bd27","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:24:03.939Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"388a176c-ca0b-493c-bded-1351d4f5bd27","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:24:03.939Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"388a176c-ca0b-493c-bded-1351d4f5bd27","podName":"raycluster-auth-token-small-group-worker-2k4hv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:23:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:24:03.939Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"388a176c-ca0b-493c-bded-1351d4f5bd27","podName":"raycluster-auth-token-small-group-worker-2k4hv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:23:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:24:03.939Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"388a176c-ca0b-493c-bded-1351d4f5bd27","name":"raycluster-auth-token","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.62","serviceIP":"10.132.0.62","podName":"raycluster-auth-token-head-xbzkj","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:23:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.62","serviceIP":"10.132.0.62","podName":"raycluster-auth-token-head-xbzkj","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:23:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:24:03.952Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"cd2e8067-4dc4-492e-89ce-11c114dea273","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:24:03.952Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"6d72e048-6148-4775-9af4-1d9ab3923f2e","rayCluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:24:03.952Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"6d72e048-6148-4775-9af4-1d9ab3923f2e","pod":"raycluster-auth-token-head-xbzkj","ip":"10.132.0.62"}
{"level":"info","ts":"2026-09-18T10:24:03.952Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"6d72e048-6148-4775-9af4-1d9ab3923f2e","pod":"raycluster-auth-token-small-group-worker-2k4hv","ip":"10.133.0.54"}
{"level":"info","ts":"2026-09-18T10:24:03.952Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"6d72e048-6148-4775-9af4-1d9ab3923f2e","rayCluster":"raycluster-auth-token","podIPs":["10.132.0.62","10.133.0.54"]}
{"level":"info","ts":"2026-09-18T10:24:03.973Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"cd2e8067-4dc4-492e-89ce-11c114dea273","name":"raycluster-auth-token-head"}
{"level":"info","ts":"2026-09-18T10:24:03.983Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"cd2e8067-4dc4-492e-89ce-11c114dea273","name":"raycluster-auth-token-workers"}
{"level":"info","ts":"2026-09-18T10:24:03.983Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"cd2e8067-4dc4-492e-89ce-11c114dea273","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:24:05.122Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"199c5ba0-94f6-40c2-a0f8-be8ceba0a9f4"}
{"level":"info","ts":"2026-09-18T10:24:05.122Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"199c5ba0-94f6-40c2-a0f8-be8ceba0a9f4"}
{"level":"info","ts":"2026-09-18T10:24:05.122Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"199c5ba0-94f6-40c2-a0f8-be8ceba0a9f4"}
{"level":"info","ts":"2026-09-18T10:24:05.122Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"199c5ba0-94f6-40c2-a0f8-be8ceba0a9f4","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:05.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"199c5ba0-94f6-40c2-a0f8-be8ceba0a9f4","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:24:05.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"199c5ba0-94f6-40c2-a0f8-be8ceba0a9f4","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:24:05.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"199c5ba0-94f6-40c2-a0f8-be8ceba0a9f4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:24:05.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"199c5ba0-94f6-40c2-a0f8-be8ceba0a9f4","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:24:05.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"199c5ba0-94f6-40c2-a0f8-be8ceba0a9f4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:24:05.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"199c5ba0-94f6-40c2-a0f8-be8ceba0a9f4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:24:05.123Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"199c5ba0-94f6-40c2-a0f8-be8ceba0a9f4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:24:05.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"199c5ba0-94f6-40c2-a0f8-be8ceba0a9f4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:24:05.123Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"199c5ba0-94f6-40c2-a0f8-be8ceba0a9f4","podName":"raycluster-auth-token-small-group-worker-2k4hv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:23:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:24:05.123Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"199c5ba0-94f6-40c2-a0f8-be8ceba0a9f4","podName":"raycluster-auth-token-small-group-worker-2k4hv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:23:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:24:05.123Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"199c5ba0-94f6-40c2-a0f8-be8ceba0a9f4","name":"raycluster-auth-token","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.62","serviceIP":"10.132.0.62","podName":"raycluster-auth-token-head-xbzkj","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:23:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:24:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.62","serviceIP":"10.132.0.62","podName":"raycluster-auth-token-head-xbzkj","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:24:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:23:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:24:05.135Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"329e6405-9308-4d04-81ee-8110baa15f1b","rayCluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:24:05.135Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"14915b08-c0c9-441c-bf60-8abd4a7ae9e4","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:24:05.135Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"329e6405-9308-4d04-81ee-8110baa15f1b","pod":"raycluster-auth-token-small-group-worker-2k4hv","ip":"10.133.0.54"}
{"level":"info","ts":"2026-09-18T10:24:05.135Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"329e6405-9308-4d04-81ee-8110baa15f1b","pod":"raycluster-auth-token-head-xbzkj","ip":"10.132.0.62"}
{"level":"info","ts":"2026-09-18T10:24:05.135Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"329e6405-9308-4d04-81ee-8110baa15f1b","rayCluster":"raycluster-auth-token","podIPs":["10.133.0.54","10.132.0.62"]}
{"level":"info","ts":"2026-09-18T10:24:05.146Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"14915b08-c0c9-441c-bf60-8abd4a7ae9e4","name":"raycluster-auth-token-head"}
{"level":"info","ts":"2026-09-18T10:24:05.157Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"14915b08-c0c9-441c-bf60-8abd4a7ae9e4","name":"raycluster-auth-token-workers"}
{"level":"info","ts":"2026-09-18T10:24:05.157Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"14915b08-c0c9-441c-bf60-8abd4a7ae9e4","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:24:05.952Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7b250654-9ac0-40af-948f-3197165c8286"}
{"level":"info","ts":"2026-09-18T10:24:05.952Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7b250654-9ac0-40af-948f-3197165c8286"}
{"level":"info","ts":"2026-09-18T10:24:05.952Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7b250654-9ac0-40af-948f-3197165c8286"}
{"level":"info","ts":"2026-09-18T10:24:05.952Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7b250654-9ac0-40af-948f-3197165c8286","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:05.952Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7b250654-9ac0-40af-948f-3197165c8286","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:24:05.952Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7b250654-9ac0-40af-948f-3197165c8286","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:24:05.952Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7b250654-9ac0-40af-948f-3197165c8286","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:24:05.952Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7b250654-9ac0-40af-948f-3197165c8286","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:24:05.952Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7b250654-9ac0-40af-948f-3197165c8286","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:24:05.952Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7b250654-9ac0-40af-948f-3197165c8286","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:24:05.952Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7b250654-9ac0-40af-948f-3197165c8286","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:24:05.952Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7b250654-9ac0-40af-948f-3197165c8286","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:24:05.952Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7b250654-9ac0-40af-948f-3197165c8286","podName":"raycluster-auth-token-small-group-worker-2k4hv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:23:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:24:05.952Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7b250654-9ac0-40af-948f-3197165c8286","podName":"raycluster-auth-token-small-group-worker-2k4hv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-09-18T10:23:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-09-18T10:24:05.952Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7b250654-9ac0-40af-948f-3197165c8286","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:24:05.952Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"7b250654-9ac0-40af-948f-3197165c8286","seconds":300}
{"level":"info","ts":"2026-09-18T10:24:14.198Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"cf416f59-70ad-45e2-bd97-ffa92bbebdbd","rayCluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:24:14.198Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"cf416f59-70ad-45e2-bd97-ffa92bbebdbd","pod":"raycluster-auth-token-head-xbzkj","ip":"10.132.0.62"}
{"level":"info","ts":"2026-09-18T10:24:14.198Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"cf416f59-70ad-45e2-bd97-ffa92bbebdbd","pod":"raycluster-auth-token-small-group-worker-2k4hv","ip":"10.133.0.54"}
{"level":"info","ts":"2026-09-18T10:24:14.198Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"cf416f59-70ad-45e2-bd97-ffa92bbebdbd","rayCluster":"raycluster-auth-token","podIPs":["10.132.0.62","10.133.0.54"]}
{"level":"info","ts":"2026-09-18T10:24:14.968Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"b06b4ef0-8b7b-4834-92cc-53d5114ef8f1"}
{"level":"info","ts":"2026-09-18T10:24:14.968Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"b06b4ef0-8b7b-4834-92cc-53d5114ef8f1"}
{"level":"info","ts":"2026-09-18T10:24:14.968Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"b06b4ef0-8b7b-4834-92cc-53d5114ef8f1"}
{"level":"info","ts":"2026-09-18T10:24:14.968Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"b06b4ef0-8b7b-4834-92cc-53d5114ef8f1","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:14.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"b06b4ef0-8b7b-4834-92cc-53d5114ef8f1","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:24:14.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"b06b4ef0-8b7b-4834-92cc-53d5114ef8f1","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:24:14.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"b06b4ef0-8b7b-4834-92cc-53d5114ef8f1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:24:14.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"b06b4ef0-8b7b-4834-92cc-53d5114ef8f1","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:24:14.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"b06b4ef0-8b7b-4834-92cc-53d5114ef8f1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:24:14.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"b06b4ef0-8b7b-4834-92cc-53d5114ef8f1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:24:14.969Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"b06b4ef0-8b7b-4834-92cc-53d5114ef8f1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:24:14.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"b06b4ef0-8b7b-4834-92cc-53d5114ef8f1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:24:14.969Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"b06b4ef0-8b7b-4834-92cc-53d5114ef8f1","name":"raycluster-auth-token","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:24:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.62","serviceIP":"10.132.0.62","podName":"raycluster-auth-token-head-xbzkj","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:24:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:23:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:24:14Z","stateTransitionTimes":{"ready":"2026-09-18T10:24:14Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.62","serviceIP":"10.132.0.62","podName":"raycluster-auth-token-head-xbzkj","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:24:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:24:14Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:23:44Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-09-18T10:23:44Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-09-18T10:24:14.982Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"c686bb65-d04e-4d7e-ad97-bcf8ca6b14a1","rayCluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:24:14.982Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"e651be87-2474-46de-ad19-bfd7f6f25ac4","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:24:14.982Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"c686bb65-d04e-4d7e-ad97-bcf8ca6b14a1","pod":"raycluster-auth-token-head-xbzkj","ip":"10.132.0.62"}
{"level":"info","ts":"2026-09-18T10:24:14.982Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"c686bb65-d04e-4d7e-ad97-bcf8ca6b14a1","pod":"raycluster-auth-token-small-group-worker-2k4hv","ip":"10.133.0.54"}
{"level":"info","ts":"2026-09-18T10:24:14.982Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"c686bb65-d04e-4d7e-ad97-bcf8ca6b14a1","rayCluster":"raycluster-auth-token","podIPs":["10.132.0.62","10.133.0.54"]}
{"level":"info","ts":"2026-09-18T10:24:14.994Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"e651be87-2474-46de-ad19-bfd7f6f25ac4","name":"raycluster-auth-token-head"}
{"level":"info","ts":"2026-09-18T10:24:15.008Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"e651be87-2474-46de-ad19-bfd7f6f25ac4","name":"raycluster-auth-token-workers"}
{"level":"info","ts":"2026-09-18T10:24:15.008Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"namespace":"test-ns-xd9fs","name":"raycluster-auth-token","reconcileID":"e651be87-2474-46de-ad19-bfd7f6f25ac4","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-09-18T10:24:16.983Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"449f3479-81b1-4ad9-9943-7603b46d2a6e"}
{"level":"info","ts":"2026-09-18T10:24:16.983Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"449f3479-81b1-4ad9-9943-7603b46d2a6e"}
{"level":"info","ts":"2026-09-18T10:24:16.983Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"449f3479-81b1-4ad9-9943-7603b46d2a6e"}
{"level":"info","ts":"2026-09-18T10:24:16.983Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"449f3479-81b1-4ad9-9943-7603b46d2a6e","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:16.983Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"449f3479-81b1-4ad9-9943-7603b46d2a6e","Found 1 head Pod":"raycluster-auth-token-head-xbzkj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-09-18T10:24:16.983Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"449f3479-81b1-4ad9-9943-7603b46d2a6e","head Pod":"raycluster-auth-token-head-xbzkj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-xbzkj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:24:16.983Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"449f3479-81b1-4ad9-9943-7603b46d2a6e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-09-18T10:24:16.983Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"449f3479-81b1-4ad9-9943-7603b46d2a6e","worker Pod":"raycluster-auth-token-small-group-worker-2k4hv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-2k4hv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-09-18T10:24:16.983Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"449f3479-81b1-4ad9-9943-7603b46d2a6e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:24:16.983Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"449f3479-81b1-4ad9-9943-7603b46d2a6e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-09-18T10:24:16.983Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"449f3479-81b1-4ad9-9943-7603b46d2a6e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-09-18T10:24:16.983Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"449f3479-81b1-4ad9-9943-7603b46d2a6e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:24:16.983Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"449f3479-81b1-4ad9-9943-7603b46d2a6e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:24:16.983Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-xd9fs"},"reconcileID":"449f3479-81b1-4ad9-9943-7603b46d2a6e","seconds":300}
{"level":"info","ts":"2026-09-18T10:24:17.990Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3c6231c1-2e7d-4a62-93f6-a880a0f80ab8"}
{"level":"info","ts":"2026-09-18T10:24:17.990Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3c6231c1-2e7d-4a62-93f6-a880a0f80ab8"}
{"level":"info","ts":"2026-09-18T10:24:17.990Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3c6231c1-2e7d-4a62-93f6-a880a0f80ab8"}
{"level":"info","ts":"2026-09-18T10:24:17.990Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3c6231c1-2e7d-4a62-93f6-a880a0f80ab8","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:17.990Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3c6231c1-2e7d-4a62-93f6-a880a0f80ab8","Found 1 head Pod":"ray-cluster-head-4f7tw","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-09-18 10:19:24 +0000 UTC,FinishedAt:2026-09-18 10:23:50 +0000 UTC,ContainerID:cri-o://29a9411dbfec80626886b53515bfc61cb577671706be03707dae47662e1dfd31,}"}
{"level":"info","ts":"2026-09-18T10:24:17.990Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3c6231c1-2e7d-4a62-93f6-a880a0f80ab8","head Pod":"ray-cluster-head-4f7tw","shouldDelete":true,"reason":"The head Pod ray-cluster-head-4f7tw status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-09-18T10:24:17.995Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3c6231c1-2e7d-4a62-93f6-a880a0f80ab8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod ray-cluster-head-4f7tw status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:17.995Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3c6231c1-2e7d-4a62-93f6-a880a0f80ab8","error":"The head Pod ray-cluster-head-4f7tw status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-09-18T10:24:17.996Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3c6231c1-2e7d-4a62-93f6-a880a0f80ab8","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:23:42Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:24:17Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:24:17Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-09-18T10:24:18.009Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3c6231c1-2e7d-4a62-93f6-a880a0f80ab8"}
{"level":"error","ts":"2026-09-18T10:24:18.009Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"3c6231c1-2e7d-4a62-93f6-a880a0f80ab8","error":"The head Pod ray-cluster-head-4f7tw status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:18.009Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"fd0a52c0-488f-4dff-b640-7022a63f2b69","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:18.009Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"6cfd2926-ee5e-4a8e-a2dd-be10e21ea67c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:18.009Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"fd0a52c0-488f-4dff-b640-7022a63f2b69","pod":"ray-cluster-head-4f7tw","ip":"10.133.0.53"}
{"level":"info","ts":"2026-09-18T10:24:18.015Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2a098dd1-1097-43f9-bbde-8bfef2376667"}
{"level":"info","ts":"2026-09-18T10:24:18.015Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2a098dd1-1097-43f9-bbde-8bfef2376667"}
{"level":"info","ts":"2026-09-18T10:24:18.015Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2a098dd1-1097-43f9-bbde-8bfef2376667"}
{"level":"info","ts":"2026-09-18T10:24:18.015Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2a098dd1-1097-43f9-bbde-8bfef2376667","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:18.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2a098dd1-1097-43f9-bbde-8bfef2376667","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-09-18T10:24:18.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2a098dd1-1097-43f9-bbde-8bfef2376667","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:24:18.015Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"6cfd2926-ee5e-4a8e-a2dd-be10e21ea67c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:18.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2a098dd1-1097-43f9-bbde-8bfef2376667","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:24:18.015Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2a098dd1-1097-43f9-bbde-8bfef2376667","namespace":"test-ns-wlgxb","name":"ray-cluster-small-group-worker-9c9dl"}
{"level":"info","ts":"2026-09-18T10:24:18.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2a098dd1-1097-43f9-bbde-8bfef2376667","The worker Pod has already been deleted":"ray-cluster-small-group-worker-9c9dl"}
{"level":"info","ts":"2026-09-18T10:24:18.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2a098dd1-1097-43f9-bbde-8bfef2376667","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:24:18.021Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2a098dd1-1097-43f9-bbde-8bfef2376667","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:24:18.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2a098dd1-1097-43f9-bbde-8bfef2376667","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:24:18.021Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2a098dd1-1097-43f9-bbde-8bfef2376667","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-09-18T10:24:18.021Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2a098dd1-1097-43f9-bbde-8bfef2376667","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-09-18T10:24:18.021Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"2a098dd1-1097-43f9-bbde-8bfef2376667","seconds":300}
{"level":"info","ts":"2026-09-18T10:24:18.021Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"fd0a52c0-488f-4dff-b640-7022a63f2b69","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-wlgxb.svc","ray-cluster-head-svc.test-ns-wlgxb.svc.cluster.local"],"ipAddresses":["10.133.0.53","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:24:18.035Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"fd0a52c0-488f-4dff-b640-7022a63f2b69","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-wlgxb.svc","ray-cluster-worker-svc.test-ns-wlgxb.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-wlgxb.svc","*.ray-cluster-worker-svc.test-ns-wlgxb.svc.cluster.local","*-worker-*.test-ns-wlgxb.svc","*-worker-*.test-ns-wlgxb.svc.cluster.local"],"ipAddresses":["10.133.0.53","127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:24:18.035Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"fd0a52c0-488f-4dff-b640-7022a63f2b69","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-09-18T10:24:18.297Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"63e6f53c-55ca-4de5-b3b7-c7ef75f539fe"}
{"level":"info","ts":"2026-09-18T10:24:18.297Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"63e6f53c-55ca-4de5-b3b7-c7ef75f539fe"}
{"level":"info","ts":"2026-09-18T10:24:18.297Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"63e6f53c-55ca-4de5-b3b7-c7ef75f539fe"}
{"level":"info","ts":"2026-09-18T10:24:18.297Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"63e6f53c-55ca-4de5-b3b7-c7ef75f539fe","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:18.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"63e6f53c-55ca-4de5-b3b7-c7ef75f539fe","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-09-18T10:24:18.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"63e6f53c-55ca-4de5-b3b7-c7ef75f539fe","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-09-18T10:24:18.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"63e6f53c-55ca-4de5-b3b7-c7ef75f539fe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-09-18T10:24:18.297Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"63e6f53c-55ca-4de5-b3b7-c7ef75f539fe","namespace":"test-ns-wlgxb","name":"ray-cluster-small-group-worker-9c9dl"}
{"level":"info","ts":"2026-09-18T10:24:18.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"63e6f53c-55ca-4de5-b3b7-c7ef75f539fe","The worker Pod has already been deleted":"ray-cluster-small-group-worker-9c9dl"}
{"level":"info","ts":"2026-09-18T10:24:18.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"63e6f53c-55ca-4de5-b3b7-c7ef75f539fe","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-09-18T10:24:18.300Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"63e6f53c-55ca-4de5-b3b7-c7ef75f539fe","group":"small-group","indices":{}}
{"level":"info","ts":"2026-09-18T10:24:18.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"63e6f53c-55ca-4de5-b3b7-c7ef75f539fe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-09-18T10:24:18.301Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"63e6f53c-55ca-4de5-b3b7-c7ef75f539fe","podName":"ray-cluster-head-4f7tw","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-09-18T10:24:18.301Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"63e6f53c-55ca-4de5-b3b7-c7ef75f539fe","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.301Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"63e6f53c-55ca-4de5-b3b7-c7ef75f539fe","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:24:17Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:24:17Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:24:18Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:24:17Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85"}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85"}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85"}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"6c3532b2-38af-4d1c-8b9b-4b7a63381666","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85"}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85"}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","enabled":true}
{"level":"info","ts":"2026-09-18T10:24:18.313Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"e9fb9d5a-bbda-40df-9e2d-d777d01af542","cluster":"ray-cluster"}
{"level":"error","ts":"2026-09-18T10:24:18.316Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:18.317Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:24:18.317Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.317Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.317Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.317Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:24:17Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:24:17Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:24:18Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:24:17Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T10:24:18Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated"}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-09-18T10:24:18.318Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"e9fb9d5a-bbda-40df-9e2d-d777d01af542","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:18.323Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"6c3532b2-38af-4d1c-8b9b-4b7a63381666","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-wlgxb.svc","ray-cluster-head-svc.test-ns-wlgxb.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:24:18.324Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-wlgxb","name":"ray-cluster"}}
{"level":"info","ts":"2026-09-18T10:24:18.324Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85"}
{"level":"error","ts":"2026-09-18T10:24:18.324Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"12cd0331-cc18-446f-bdd9-3fcc73312b85","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:18.329Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d"}
{"level":"info","ts":"2026-09-18T10:24:18.329Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d"}
{"level":"info","ts":"2026-09-18T10:24:18.329Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d"}
{"level":"info","ts":"2026-09-18T10:24:18.329Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:18.329Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d"}
{"level":"info","ts":"2026-09-18T10:24:18.329Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:18.329Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d"}
{"level":"info","ts":"2026-09-18T10:24:18.329Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:18.329Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:18.329Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:24:18.329Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:24:18.329Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:24:18.329Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:18.329Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:18.329Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","enabled":true}
{"level":"error","ts":"2026-09-18T10:24:18.333Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:18.333Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:24:18.333Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.333Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.333Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.333Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:24:18Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","podName":"ray-cluster-head-4f7tw","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:24:17Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-09-18T10:24:18Z","stateTransitionTimes":{"ready":"2026-09-18T10:19:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-09-18T10:17:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-09-18T10:24:17Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-09-18T10:19:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-09-18T10:17:51Z","reason":"RayClusterSuspending","message":""},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-09-18T10:24:18Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated"}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-09-18T10:24:18.337Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"6c3532b2-38af-4d1c-8b9b-4b7a63381666","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-wlgxb.svc","ray-cluster-worker-svc.test-ns-wlgxb.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-wlgxb.svc","*.ray-cluster-worker-svc.test-ns-wlgxb.svc.cluster.local","*-worker-*.test-ns-wlgxb.svc","*-worker-*.test-ns-wlgxb.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-09-18T10:24:18.337Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"6c3532b2-38af-4d1c-8b9b-4b7a63381666"}
{"level":"info","ts":"2026-09-18T10:24:18.347Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a633e447-9216-4231-b3d9-c0c1a4066217","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:18.347Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"a633e447-9216-4231-b3d9-c0c1a4066217"}
{"level":"info","ts":"2026-09-18T10:24:18.347Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c62f22fc-c128-4a57-b633-3ddbc04b29b9","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:18.365Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d"}
{"level":"error","ts":"2026-09-18T10:24:18.365Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"8bc5e113-8320-4e61-ab2d-2c6c2c63658d","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:18.368Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c62f22fc-c128-4a57-b633-3ddbc04b29b9","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:18.376Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b"}
{"level":"info","ts":"2026-09-18T10:24:18.376Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b"}
{"level":"info","ts":"2026-09-18T10:24:18.376Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b"}
{"level":"info","ts":"2026-09-18T10:24:18.376Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:18.376Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b"}
{"level":"info","ts":"2026-09-18T10:24:18.376Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:18.376Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b"}
{"level":"info","ts":"2026-09-18T10:24:18.376Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:18.376Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:18.376Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:24:18.376Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:24:18.376Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:24:18.376Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:18.376Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:18.377Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b","enabled":true}
{"level":"error","ts":"2026-09-18T10:24:18.380Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:18.380Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:24:18.381Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.381Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.381Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.381Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b"}
{"level":"error","ts":"2026-09-18T10:24:18.381Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"bb329b0e-3d6e-46ec-ad9f-a0729fc6b96b","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:18.401Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392"}
{"level":"info","ts":"2026-09-18T10:24:18.401Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392"}
{"level":"info","ts":"2026-09-18T10:24:18.401Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392"}
{"level":"info","ts":"2026-09-18T10:24:18.401Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:18.402Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392"}
{"level":"info","ts":"2026-09-18T10:24:18.402Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:18.402Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392"}
{"level":"info","ts":"2026-09-18T10:24:18.402Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:18.402Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:18.402Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:24:18.402Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:24:18.402Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:24:18.402Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:18.402Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:18.402Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392","enabled":true}
{"level":"error","ts":"2026-09-18T10:24:18.405Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:18.405Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:24:18.405Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.405Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.405Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.405Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392"}
{"level":"error","ts":"2026-09-18T10:24:18.405Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f07aea52-4702-4161-b58f-62c240aa3392","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:18.446Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131"}
{"level":"info","ts":"2026-09-18T10:24:18.446Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131"}
{"level":"info","ts":"2026-09-18T10:24:18.446Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131"}
{"level":"info","ts":"2026-09-18T10:24:18.446Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:18.446Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131"}
{"level":"info","ts":"2026-09-18T10:24:18.446Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:18.446Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131"}
{"level":"info","ts":"2026-09-18T10:24:18.446Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:18.446Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:18.446Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:24:18.446Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:24:18.446Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:24:18.446Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:18.446Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:18.446Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131","enabled":true}
{"level":"error","ts":"2026-09-18T10:24:18.450Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:18.450Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:24:18.450Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.450Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.450Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.450Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131"}
{"level":"error","ts":"2026-09-18T10:24:18.450Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"27261dd3-80dd-44d5-9412-2e09f697c131","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:18.531Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d"}
{"level":"info","ts":"2026-09-18T10:24:18.531Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d"}
{"level":"info","ts":"2026-09-18T10:24:18.531Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d"}
{"level":"info","ts":"2026-09-18T10:24:18.531Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:18.531Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d"}
{"level":"info","ts":"2026-09-18T10:24:18.531Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:18.531Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d"}
{"level":"info","ts":"2026-09-18T10:24:18.531Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:18.531Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:18.531Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:24:18.531Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:24:18.531Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:24:18.531Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:18.531Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:18.531Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d","enabled":true}
{"level":"error","ts":"2026-09-18T10:24:18.535Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:18.535Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:24:18.535Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.535Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.535Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.535Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d"}
{"level":"error","ts":"2026-09-18T10:24:18.535Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"54b6bb2f-b08c-41de-a163-339a8f4b666d","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:18.696Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d"}
{"level":"info","ts":"2026-09-18T10:24:18.696Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d"}
{"level":"info","ts":"2026-09-18T10:24:18.696Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d"}
{"level":"info","ts":"2026-09-18T10:24:18.696Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:18.696Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d"}
{"level":"info","ts":"2026-09-18T10:24:18.696Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:18.696Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d"}
{"level":"info","ts":"2026-09-18T10:24:18.696Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:18.696Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:18.696Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:24:18.696Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:24:18.696Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:24:18.696Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:18.696Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:18.696Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d","enabled":true}
{"level":"error","ts":"2026-09-18T10:24:18.700Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:18.700Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:24:18.700Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.700Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.700Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:18.700Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d"}
{"level":"error","ts":"2026-09-18T10:24:18.700Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"ca38f6a8-d313-452b-958a-95d3a7b0c39d","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:19.021Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7"}
{"level":"info","ts":"2026-09-18T10:24:19.021Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7"}
{"level":"info","ts":"2026-09-18T10:24:19.021Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7"}
{"level":"info","ts":"2026-09-18T10:24:19.021Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:19.021Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7"}
{"level":"info","ts":"2026-09-18T10:24:19.021Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:19.021Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7"}
{"level":"info","ts":"2026-09-18T10:24:19.021Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:19.021Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:19.021Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:24:19.021Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:24:19.021Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:24:19.021Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:19.021Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:19.021Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7","enabled":true}
{"level":"error","ts":"2026-09-18T10:24:19.025Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:19.025Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:24:19.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:19.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:19.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:19.025Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7"}
{"level":"error","ts":"2026-09-18T10:24:19.025Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"f6fec21a-057a-425e-832a-b5fcffe2dfc7","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:19.666Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a"}
{"level":"info","ts":"2026-09-18T10:24:19.666Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a"}
{"level":"info","ts":"2026-09-18T10:24:19.666Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a"}
{"level":"info","ts":"2026-09-18T10:24:19.666Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:19.666Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a"}
{"level":"info","ts":"2026-09-18T10:24:19.666Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:19.666Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a"}
{"level":"info","ts":"2026-09-18T10:24:19.666Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:19.666Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:19.666Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:24:19.666Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:24:19.666Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:24:19.666Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:19.666Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:19.666Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a","enabled":true}
{"level":"error","ts":"2026-09-18T10:24:19.670Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:19.670Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:24:19.670Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:19.670Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:19.670Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:19.670Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a"}
{"level":"error","ts":"2026-09-18T10:24:19.671Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"79bfd498-844a-44e5-b511-8931efb0853a","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:20.952Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78"}
{"level":"info","ts":"2026-09-18T10:24:20.952Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78"}
{"level":"info","ts":"2026-09-18T10:24:20.952Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78"}
{"level":"info","ts":"2026-09-18T10:24:20.952Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:20.952Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78"}
{"level":"info","ts":"2026-09-18T10:24:20.952Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:20.952Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78"}
{"level":"info","ts":"2026-09-18T10:24:20.952Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:20.952Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:20.952Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:24:20.952Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:24:20.952Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:24:20.952Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:20.952Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:20.952Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78","enabled":true}
{"level":"error","ts":"2026-09-18T10:24:20.956Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:20.956Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:24:20.956Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:20.956Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:20.956Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:20.956Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78"}
{"level":"error","ts":"2026-09-18T10:24:20.956Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"1c9f7946-8247-49d8-880a-784be4ed5c78","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:21.763Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c819757b-e944-4adb-ad11-bdf4dc4251d7","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:21.763Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"c819757b-e944-4adb-ad11-bdf4dc4251d7"}
{"level":"info","ts":"2026-09-18T10:24:23.517Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d"}
{"level":"info","ts":"2026-09-18T10:24:23.517Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d"}
{"level":"info","ts":"2026-09-18T10:24:23.517Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d"}
{"level":"info","ts":"2026-09-18T10:24:23.517Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-09-18T10:24:23.517Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d"}
{"level":"info","ts":"2026-09-18T10:24:23.517Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:23.517Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d"}
{"level":"info","ts":"2026-09-18T10:24:23.517Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:23.517Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:23.517Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-09-18T10:24:23.517Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-09-18T10:24:23.517Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-09-18T10:24:23.517Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:23.517Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-09-18T10:24:23.517Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d","enabled":true}
{"level":"error","ts":"2026-09-18T10:24:23.521Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:332\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:126\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:23.521Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated"}
{"level":"info","ts":"2026-09-18T10:24:23.521Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:23.521Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:23.521Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d","filter labels":["test-ns-wlgxb",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-09-18T10:24:23.521Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d"}
{"level":"error","ts":"2026-09-18T10:24:23.521Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"fecb1adc-6679-4d42-860a-54ac31af7b2d","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:24.924Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"694427ac-8941-42da-af0d-bea07013655e","namespacedName":{"name":"ray-cluster","namespace":"test-ns-wlgxb"}}
{"level":"info","ts":"2026-09-18T10:24:24.924Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"694427ac-8941-42da-af0d-bea07013655e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:24.924Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"694427ac-8941-42da-af0d-bea07013655e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:24.924Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"07de0905-3c98-4e91-ac5b-5ef18cfe4a84"}
{"level":"info","ts":"2026-09-18T10:24:24.924Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"694427ac-8941-42da-af0d-bea07013655e","namespace":"test-ns-wlgxb","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T10:24:24.924Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"e1481c41-0a80-4c61-a951-7e76717b14f9","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:24.924Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"e1481c41-0a80-4c61-a951-7e76717b14f9","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:24.924Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8eada8e6-2e9d-4432-8760-6837691c347a"}
{"level":"error","ts":"2026-09-18T10:24:24.927Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"e1481c41-0a80-4c61-a951-7e76717b14f9","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:24:24.927Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"e1481c41-0a80-4c61-a951-7e76717b14f9","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:24.927Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"e1481c41-0a80-4c61-a951-7e76717b14f9"}
{"level":"error","ts":"2026-09-18T10:24:24.927Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"e1481c41-0a80-4c61-a951-7e76717b14f9","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:24.930Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"694427ac-8941-42da-af0d-bea07013655e","name":"kuberay-gateway-access","namespace":"test-ns-wlgxb"}
{"level":"info","ts":"2026-09-18T10:24:24.933Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8361a593-efff-4e8e-9558-7709919bb1b9","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:24.933Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8361a593-efff-4e8e-9558-7709919bb1b9","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:24.935Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"694427ac-8941-42da-af0d-bea07013655e","name":"test-ns-wlgxb-ray-cluster","namespace":"opendatahub"}
{"level":"error","ts":"2026-09-18T10:24:24.936Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8361a593-efff-4e8e-9558-7709919bb1b9","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:24:24.936Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8361a593-efff-4e8e-9558-7709919bb1b9","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:24.936Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8361a593-efff-4e8e-9558-7709919bb1b9"}
{"level":"error","ts":"2026-09-18T10:24:24.936Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"8361a593-efff-4e8e-9558-7709919bb1b9","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:24.939Z","logger":"authentication-controller","msg":"Failed to delete ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"694427ac-8941-42da-af0d-bea07013655e","error":"configmaps \"kube-rbac-proxy-config-ray-cluster\" not found"}
{"level":"info","ts":"2026-09-18T10:24:24.944Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"694427ac-8941-42da-af0d-bea07013655e","serviceAccount":"ray-cluster-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-09-18T10:24:24.947Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"caf7a6a7-24ef-448a-8c02-f1db285a0d1f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:24.947Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"caf7a6a7-24ef-448a-8c02-f1db285a0d1f","rayCluster":"ray-cluster"}
{"level":"error","ts":"2026-09-18T10:24:24.950Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"caf7a6a7-24ef-448a-8c02-f1db285a0d1f","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-09-18T10:24:24.950Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"caf7a6a7-24ef-448a-8c02-f1db285a0d1f","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:24.950Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"caf7a6a7-24ef-448a-8c02-f1db285a0d1f"}
{"level":"error","ts":"2026-09-18T10:24:24.950Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"caf7a6a7-24ef-448a-8c02-f1db285a0d1f","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-wlgxb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:24.951Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:24.951Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-wlgxb"}
{"level":"info","ts":"2026-09-18T10:24:24.954Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"reconcileID":"cd98d859-6013-43df-be01-50b65c943de6"}
{"level":"info","ts":"2026-09-18T10:24:24.963Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"694427ac-8941-42da-af0d-bea07013655e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:24.963Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9bcb0f66-a7d1-4f35-a097-6f82b48864cd","namespacedName":{"name":"ray-cluster","namespace":"test-ns-wlgxb"}}
{"level":"info","ts":"2026-09-18T10:24:24.963Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9bcb0f66-a7d1-4f35-a097-6f82b48864cd","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:24.963Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9bcb0f66-a7d1-4f35-a097-6f82b48864cd","cluster":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:24.963Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9bcb0f66-a7d1-4f35-a097-6f82b48864cd","namespace":"test-ns-wlgxb","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-09-18T10:24:24.963Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9bcb0f66-a7d1-4f35-a097-6f82b48864cd","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-09-18T10:24:24.964Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"2f530912-fc31-4c43-95c9-6fd2bec3727a"}
{"level":"info","ts":"2026-09-18T10:24:24.964Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ef50809d-8c11-4370-94bd-e7b5dc402f9a","namespace":"test-ns-wlgxb","clusterName":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:24.968Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ef500f65-360a-4e4c-ad9f-73cdfb940da2"}
{"level":"error","ts":"2026-09-18T10:24:24.968Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9bcb0f66-a7d1-4f35-a097-6f82b48864cd","error":"rayclusters.ray.io \"ray-cluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:196\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:137\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:24.969Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9bcb0f66-a7d1-4f35-a097-6f82b48864cd"}
{"level":"error","ts":"2026-09-18T10:24:24.969Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"9bcb0f66-a7d1-4f35-a097-6f82b48864cd","error":"rayclusters.ray.io \"ray-cluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:24.969Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"b3f533de-88f7-4f0a-b98d-d237b374358b","namespacedName":{"name":"ray-cluster","namespace":"test-ns-wlgxb"}}
{"level":"info","ts":"2026-09-18T10:24:24.969Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"b3f533de-88f7-4f0a-b98d-d237b374358b"}
{"level":"info","ts":"2026-09-18T10:24:24.969Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"b3f533de-88f7-4f0a-b98d-d237b374358b","namespace":"test-ns-wlgxb","authClusters":0}
{"level":"info","ts":"2026-09-18T10:24:24.969Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"b3f533de-88f7-4f0a-b98d-d237b374358b","namespace":"test-ns-wlgxb"}
{"level":"info","ts":"2026-09-18T10:24:24.974Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"acff785d-51f5-4db5-a10c-49a1403d56e3","namespacedName":{"name":"ray-cluster","namespace":"test-ns-wlgxb"}}
{"level":"info","ts":"2026-09-18T10:24:24.974Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"acff785d-51f5-4db5-a10c-49a1403d56e3"}
{"level":"info","ts":"2026-09-18T10:24:24.974Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"acff785d-51f5-4db5-a10c-49a1403d56e3","namespace":"test-ns-wlgxb","authClusters":0}
{"level":"info","ts":"2026-09-18T10:24:24.974Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"acff785d-51f5-4db5-a10c-49a1403d56e3","namespace":"test-ns-wlgxb"}
{"level":"error","ts":"2026-09-18T10:24:24.975Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ef50809d-8c11-4370-94bd-e7b5dc402f9a","issuer":"ray-selfsigned-issuer-ray-cluster","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:24.979Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"1a37ba2a-ce2e-4c1b-9226-e1eb68edbd5e","namespacedName":{"name":"ray-cluster","namespace":"test-ns-wlgxb"}}
{"level":"info","ts":"2026-09-18T10:24:24.979Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"1a37ba2a-ce2e-4c1b-9226-e1eb68edbd5e"}
{"level":"info","ts":"2026-09-18T10:24:24.979Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"1a37ba2a-ce2e-4c1b-9226-e1eb68edbd5e","namespace":"test-ns-wlgxb","authClusters":0}
{"level":"info","ts":"2026-09-18T10:24:24.979Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"1a37ba2a-ce2e-4c1b-9226-e1eb68edbd5e","namespace":"test-ns-wlgxb"}
{"level":"info","ts":"2026-09-18T10:24:24.987Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"03cb618f-8842-43fa-b13c-6c10d734ca66"}
{"level":"info","ts":"2026-09-18T10:24:25.022Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ef50809d-8c11-4370-94bd-e7b5dc402f9a","errorCount":1}
{"level":"info","ts":"2026-09-18T10:24:25.022Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ef50809d-8c11-4370-94bd-e7b5dc402f9a"}
{"level":"error","ts":"2026-09-18T10:24:25.022Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"ef50809d-8c11-4370-94bd-e7b5dc402f9a","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:25.022Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"153aa366-4c42-4777-a3b7-ebdd1688f2eb","namespace":"test-ns-wlgxb","clusterName":"ray-cluster"}
{"level":"error","ts":"2026-09-18T10:24:25.026Z","msg":"Failed to delete CA secret","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"153aa366-4c42-4777-a3b7-ebdd1688f2eb","secret":"ray-cluster-ca-secret-16e29b1a","error":"secrets \"ray-cluster-ca-secret-16e29b1a\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:301\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:25.026Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"153aa366-4c42-4777-a3b7-ebdd1688f2eb","errorCount":1}
{"level":"info","ts":"2026-09-18T10:24:25.026Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"153aa366-4c42-4777-a3b7-ebdd1688f2eb"}
{"level":"error","ts":"2026-09-18T10:24:25.026Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"153aa366-4c42-4777-a3b7-ebdd1688f2eb","error":"secrets \"ray-cluster-ca-secret-16e29b1a\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-09-18T10:24:25.063Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"68f7deb2-6125-48e1-a8e6-c3e84c15e898","namespace":"test-ns-wlgxb","clusterName":"ray-cluster"}
{"level":"info","ts":"2026-09-18T10:24:25.063Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-wlgxb"},"namespace":"test-ns-wlgxb","name":"ray-cluster","reconcileID":"68f7deb2-6125-48e1-a8e6-c3e84c15e898","namespace":"test-ns-wlgxb","clusterName":"ray-cluster"}